diff --git a/.github/workflows/phase4-payloads.yml b/.github/workflows/phase4-payloads.yml new file mode 100644 index 0000000..37536fa --- /dev/null +++ b/.github/workflows/phase4-payloads.yml @@ -0,0 +1,514 @@ +name: Phase 4 node toolkit and Celestia payloads + +on: + pull_request: + paths: + - '.github/workflows/phase4-payloads.yml' + - 'catalog/components/**' + - 'evidence/phase4/**' + - 'scripts/phase4_*' + - 'scripts/probe_phase4_toolkit.sh' + - 'scripts/manifest_single_file.py' + - 'scripts/emit_phase4_evidence.py' + - 'scripts/inspect_macos_signature.py' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: phase4-payloads-${{ github.ref }} + cancel-in-progress: false + +jobs: + toolkit-linux: + name: Toolkit ${{ matrix.os }}/${{ matrix.arch }} official-byte repack + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: amd64 + runner: ubuntu-24.04 + source_name: midnight-node-toolkit-2.0.0-rc.4-linux-amd64.tar.gz + source_size: 50581636 + source_sha256: fdd9f6974c1c20928d3b9dc79c13c51bf984eb6dc4cdaf1978a1b037794402de + component: catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json + - os: linux + arch: arm64 + runner: ubuntu-24.04-arm + source_name: midnight-node-toolkit-2.0.0-rc.4-linux-arm64.tar.gz + source_size: 48937006 + source_sha256: 85513afbfe4dd3fccadde6ad5d7ccf82b8fa0c88d010445fc62c223c9ef078eb + component: catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json + runs-on: ${{ matrix.runner }} + timeout-minutes: 75 + steps: + - name: Check out exact forge source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Repack twice and run native contract + env: + SOURCE_NAME: ${{ matrix.source_name }} + SOURCE_SIZE: ${{ matrix.source_size }} + SOURCE_SHA256: ${{ matrix.source_sha256 }} + TARGET_OS: ${{ matrix.os }} + TARGET_ARCH: ${{ matrix.arch }} + COMPONENT: ${{ matrix.component }} + run: | + set -euo pipefail + python3 scripts/check_runner_capability.py --runner-label "${{ matrix.runner }}" --expected-os "$TARGET_OS" --expected-arch "$TARGET_ARCH" --require-tool git --min-free-gib 5 + python3 scripts/validate_phase4_contract.py --root . + work="$RUNNER_TEMP/phase4-toolkit-$TARGET_ARCH" + out="$GITHUB_WORKSPACE/phase4-output-toolkit-$TARGET_OS-$TARGET_ARCH" + mkdir -p "$work/download" "$work/run1/tree" "$work/run2/tree" "$work/run1/evidence" "$work/run2/evidence" "$out/payloads" "$out/evidence" "$out/sbom" + python3 scripts/fetch_verified.py \ + --url "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/$SOURCE_NAME" \ + --output "$work/download/$SOURCE_NAME" --size "$SOURCE_SIZE" --sha256 "$SOURCE_SHA256" + for run in run1 run2; do + python3 scripts/phase4_payloads.py --contract toolkit --archive "$work/download/$SOURCE_NAME" \ + --staging "$work/$run/tree" --member-manifest "$work/$run/evidence/member-manifest.json" \ + --report "$work/$run/evidence/source-inspection.json" + python3 scripts/package_deterministic.py --input-dir "$work/$run/tree" \ + --members "$work/$run/evidence/member-manifest.json" \ + --output "$work/$run/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" + done + test "$(sha256sum "$work/run1/tree/midnight-node-toolkit" | cut -d' ' -f1)" = "$(sha256sum "$work/run2/tree/midnight-node-toolkit" | cut -d' ' -f1)" + test "$(sha256sum "$work/run1/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" | cut -d' ' -f1)" = "$(sha256sum "$work/run2/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" | cut -d' ' -f1)" + cp "$work/run1/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" "$out/payloads/" + cp "$work/run1/evidence/member-manifest.json" "$out/evidence/member-manifest.json" + cp "$work/run1/evidence/source-inspection.json" "$out/evidence/source-inspection.json" + jq '{schemaVersion:"archive-policy-v1",container:"zip",maxCompressedBytes:100663296,maxExpandedBytes:268435456,maxMembers:1,maxExpansionRatio:8,expectedMembers:.members}' \ + "$out/evidence/member-manifest.json" >"$work/archive-policy.json" + python3 scripts/validate_archive.py --archive "$out/payloads/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" \ + --policy "$work/archive-policy.json" --scratch-parent "$RUNNER_TEMP" + python3 scripts/validate_native.py --binary "$work/run1/tree/midnight-node-toolkit" \ + --os "$TARGET_OS" --arch "$TARGET_ARCH" --runner-os "$TARGET_OS" --runner-arch "$TARGET_ARCH" + { + uname -a + cat /etc/os-release + file "$work/run1/tree/midnight-node-toolkit" + readelf -h "$work/run1/tree/midnight-node-toolkit" + readelf -l "$work/run1/tree/midnight-node-toolkit" + readelf -d "$work/run1/tree/midnight-node-toolkit" + readelf --version-info "$work/run1/tree/midnight-node-toolkit" + sha256sum "$work/run1/tree/midnight-node-toolkit" "$work/run1/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" + } >"$out/evidence/system.log" 2>&1 + bash scripts/probe_phase4_toolkit.sh "$work/run1/tree/midnight-node-toolkit" "$work/probes" + for probe in "$work/probes"/*; do printf '\n== %s ==\n' "$(basename "$probe")"; cat "$probe"; done >"$out/evidence/probe.log" + bash scripts/phase4_sbom.sh "$work/run1/tree/midnight-node-toolkit" "$out/sbom/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4" "$TARGET_OS" "$TARGET_ARCH" + python3 scripts/emit_phase4_evidence.py --component "$COMPONENT" \ + --payload "$out/payloads/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.zip" \ + --member-manifest "$out/evidence/member-manifest.json" --source-report "$out/evidence/source-inspection.json" \ + --probe-log "$out/evidence/probe.log" --system-log "$out/evidence/system.log" \ + --sbom-spdx "$out/sbom/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.spdx.json" \ + --sbom-cyclonedx "$out/sbom/midnight-node-toolkit-$TARGET_OS-$TARGET_ARCH-2.0.0-rc.4.cyclonedx.json" \ + --output "$out/evidence/payload-evidence.json" --checksums "$out/evidence/SHA256SUMS" + - name: Upload inert verified toolkit payload and evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase4-toolkit-${{ matrix.os }}-${{ matrix.arch }} + path: phase4-output-toolkit-${{ matrix.os }}-${{ matrix.arch }} + if-no-files-found: error + compression-level: 0 + retention-days: 30 + + node-linux-arm64: + name: Node linux/arm64 deterministic repack and start + runs-on: ubuntu-24.04-arm + timeout-minutes: 90 + steps: + - name: Check out exact forge source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Repack twice and run node with and without res + run: | + set -euo pipefail + python3 scripts/check_runner_capability.py --runner-label ubuntu-24.04-arm --expected-os linux --expected-arch arm64 --require-tool git --min-free-gib 8 + python3 scripts/validate_phase4_contract.py --root . + work="$RUNNER_TEMP/phase4-node-arm64" + out="$GITHUB_WORKSPACE/phase4-output-node-linux-arm64" + source_name=midnight-node-2.0.0-rc.4-linux-arm64.tar.gz + payload_name=midnight-node-linux-arm64-2.0.0-rc.4.zip + executable_name=midnight-node-linux-arm64-2.0.0-rc.4 + mkdir -p "$work/download" "$work/run1/tree" "$work/run2/tree" "$work/run1/evidence" "$work/run2/evidence" "$out/payloads" "$out/evidence" "$out/sbom" + python3 scripts/fetch_verified.py \ + --url "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/$source_name" \ + --output "$work/download/$source_name" --size 83084349 \ + --sha256 2bfd0f768a88d0d1d49cc97bfd6b9f4401646458887219e8c021bff8d9a9ae0b + for run in run1 run2; do + python3 scripts/phase4_payloads.py --contract node --archive "$work/download/$source_name" \ + --renamed-executable "$executable_name" --staging "$work/$run/tree" \ + --member-manifest "$work/$run/evidence/member-manifest.json" --report "$work/$run/evidence/source-inspection.json" + python3 scripts/package_deterministic.py --input-dir "$work/$run/tree" \ + --members "$work/$run/evidence/member-manifest.json" --output "$work/$run/$payload_name" + done + test "$(sha256sum "$work/run1/$payload_name" | cut -d' ' -f1)" = "$(sha256sum "$work/run2/$payload_name" | cut -d' ' -f1)" + cp "$work/run1/$payload_name" "$out/payloads/$payload_name" + cp "$work/run1/evidence/member-manifest.json" "$out/evidence/member-manifest.json" + cp "$work/run1/evidence/source-inspection.json" "$out/evidence/source-inspection.json" + jq '{schemaVersion:"archive-policy-v1",container:"zip",maxCompressedBytes:134217728,maxExpandedBytes:536870912,maxMembers:512,maxExpansionRatio:8,expectedMembers:.members}' \ + "$out/evidence/member-manifest.json" >"$work/archive-policy.json" + python3 scripts/validate_archive.py --archive "$out/payloads/$payload_name" --policy "$work/archive-policy.json" --scratch-parent "$RUNNER_TEMP" + python3 scripts/validate_native.py --binary "$work/run1/tree/$executable_name" --os linux --arch arm64 --runner-os linux --runner-arch arm64 + port=$(shuf -i 10000-60000 -n 1) + expected_node_cwd=$(realpath "$work/run1/tree") + env --chdir="$expected_node_cwd" CFG_PRESET=dev "./$executable_name" --tmp --rpc-port "$port" \ + >"$work/node.stdout" 2>"$work/node.stderr" & + node_pid=$! + printf '%s\n' "$node_pid" >"$work/node.pid" + cleanup_node() { + if [[ -n "${node_pid:-}" ]] && kill -0 "$node_pid" 2>/dev/null; then + kill -TERM "$node_pid" || true + wait "$node_pid" || true + fi + } + trap cleanup_node EXIT + actual_node_cwd= + for _ in $(seq 1 20); do + actual_node_cwd=$(readlink -e "/proc/$node_pid/cwd" || true) + if [[ -n "$actual_node_cwd" ]]; then break; fi + sleep 0.1 + done + if [[ "$actual_node_cwd" != "$expected_node_cwd" ]]; then cat "$work/node.stderr" >&2; exit 2; fi + printf 'pid=%s\nexpectedCwd=%s\nactualCwd=%s\n' "$node_pid" "$expected_node_cwd" "$actual_node_cwd" >"$work/node-cwd.txt" + for _ in $(seq 1 60); do + if curl --silent --fail --header 'content-type: application/json' \ + --data '{"id":1,"jsonrpc":"2.0","method":"system_health","params":[]}' \ + "http://127.0.0.1:$port" >"$work/node-health.json"; then break; fi + if ! kill -0 "$(cat "$work/node.pid")" 2>/dev/null; then cat "$work/node.stderr" >&2; exit 2; fi + sleep 1 + done + jq -e '.jsonrpc=="2.0" and (.result|type=="object")' "$work/node-health.json" >/dev/null + kill -TERM "$node_pid" + wait "$node_pid" || true + node_pid= + trap - EXIT + mkdir -p "$work/missing-res" + cp "$work/run1/tree/$executable_name" "$work/missing-res/$executable_name" + set +e + (cd "$work/missing-res" && timeout 20s env CFG_PRESET=dev "./$executable_name" --tmp --rpc-port "$((port + 1))") >"$work/missing-res.stdout" 2>"$work/missing-res.stderr" + missing_status=$? + set -e + test "$missing_status" -ne 0 + test "$missing_status" -ne 124 + grep -Eiq 'res|chain|spec|No such file' "$work/missing-res.stderr" + (cd "$work/run1/tree" && "./$executable_name" --version) >"$work/version.txt" + (cd "$work/run1/tree" && "./$executable_name" --help) >"$work/help.txt" + { + uname -a + cat /etc/os-release + file "$work/run1/tree/$executable_name" + readelf -h "$work/run1/tree/$executable_name" + readelf -l "$work/run1/tree/$executable_name" + readelf -d "$work/run1/tree/$executable_name" + readelf --version-info "$work/run1/tree/$executable_name" + sha256sum "$work/run1/tree/$executable_name" "$out/payloads/$payload_name" + } >"$out/evidence/system.log" 2>&1 + { + cat "$work/node-cwd.txt" "$work/version.txt" "$work/help.txt" "$work/node-health.json" "$work/node.stdout" "$work/node.stderr" + printf '\nmissingResExit=%s\n' "$missing_status" + cat "$work/missing-res.stdout" "$work/missing-res.stderr" + } >"$out/evidence/probe.log" + bash scripts/phase4_sbom.sh "$work/run1/tree/$executable_name" "$out/sbom/$executable_name" linux arm64 + python3 scripts/emit_phase4_evidence.py --component catalog/components/midnight-node-2.0.0-rc.4-linux-arm64.json \ + --payload "$out/payloads/$payload_name" --member-manifest "$out/evidence/member-manifest.json" \ + --source-report "$out/evidence/source-inspection.json" --probe-log "$out/evidence/probe.log" \ + --system-log "$out/evidence/system.log" --sbom-spdx "$out/sbom/$executable_name.spdx.json" \ + --sbom-cyclonedx "$out/sbom/$executable_name.cyclonedx.json" \ + --output "$out/evidence/payload-evidence.json" --checksums "$out/evidence/SHA256SUMS" + - name: Upload inert verified node payload and evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase4-node-linux-arm64 + path: phase4-output-node-linux-arm64 + if-no-files-found: error + compression-level: 0 + retention-days: 30 + + celestia-linux-arm64: + name: ${{ matrix.family }} linux/arm64 identity mirror + strategy: + fail-fast: false + matrix: + include: + - family: celestia-appd + contract: celestia-appd + source_name: celestia-app_Linux_arm64.tar.gz + output_name: celestia-appd-linux-arm64-v6.4.10.tar.gz + size: 180685852 + sha256: 52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b + url: https://github.com/celestiaorg/celestia-app/releases/download/v6.4.10/celestia-app_Linux_arm64.tar.gz + executable: celestia-appd + component: catalog/components/celestia-appd-6.4.10-linux-arm64.json + - family: celestia-node + contract: celestia-node + source_name: celestia-node_Linux_arm64.tar.gz + output_name: celestia-node-linux-arm64-v0.28.4.tar.gz + size: 71184641 + sha256: 09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c + url: https://github.com/celestiaorg/celestia-node/releases/download/v0.28.4/celestia-node_Linux_arm64.tar.gz + executable: celestia + component: catalog/components/celestia-node-0.28.4-linux-arm64.json + runs-on: ubuntu-24.04-arm + timeout-minutes: 75 + steps: + - name: Check out exact forge source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Preserve upstream bytes and verify native three-member contract + env: + FAMILY: ${{ matrix.family }} + CONTRACT: ${{ matrix.contract }} + OUTPUT_NAME: ${{ matrix.output_name }} + SOURCE_URL: ${{ matrix.url }} + SOURCE_SIZE: ${{ matrix.size }} + SOURCE_SHA256: ${{ matrix.sha256 }} + EXECUTABLE: ${{ matrix.executable }} + COMPONENT: ${{ matrix.component }} + run: | + set -euo pipefail + python3 scripts/check_runner_capability.py --runner-label ubuntu-24.04-arm --expected-os linux --expected-arch arm64 --require-tool git --min-free-gib 6 + work="$RUNNER_TEMP/phase4-$FAMILY" + out="$GITHUB_WORKSPACE/phase4-output-$FAMILY-linux-arm64" + mkdir -p "$work/tree" "$out/payloads" "$out/evidence" "$out/sbom" + python3 scripts/fetch_verified.py --url "$SOURCE_URL" --output "$out/payloads/$OUTPUT_NAME" --size "$SOURCE_SIZE" --sha256 "$SOURCE_SHA256" + test "$(sha256sum "$out/payloads/$OUTPUT_NAME" | cut -d' ' -f1)" = "$SOURCE_SHA256" + python3 scripts/phase4_payloads.py --contract "$CONTRACT" --archive "$out/payloads/$OUTPUT_NAME" \ + --staging "$work/tree" --member-manifest "$out/evidence/member-manifest.json" --report "$out/evidence/source-inspection.json" + python3 scripts/validate_native.py --binary "$work/tree/$EXECUTABLE" --os linux --arch arm64 --runner-os linux --runner-arch arm64 + "$work/tree/$EXECUTABLE" version >"$work/version.txt" 2>&1 + "$work/tree/$EXECUTABLE" --help >"$work/help.txt" 2>&1 + { + uname -a + cat /etc/os-release + file "$work/tree/$EXECUTABLE" + readelf -h "$work/tree/$EXECUTABLE" + readelf -l "$work/tree/$EXECUTABLE" + readelf -d "$work/tree/$EXECUTABLE" + readelf --version-info "$work/tree/$EXECUTABLE" + sha256sum "$out/payloads/$OUTPUT_NAME" "$work/tree/LICENSE" "$work/tree/README.md" "$work/tree/$EXECUTABLE" + } >"$out/evidence/system.log" 2>&1 + cat "$work/version.txt" "$work/help.txt" >"$out/evidence/probe.log" + bash scripts/phase4_sbom.sh "$work/tree/$EXECUTABLE" "$out/sbom/${OUTPUT_NAME%.tar.gz}" linux arm64 + python3 scripts/emit_phase4_evidence.py --component "$COMPONENT" --payload "$out/payloads/$OUTPUT_NAME" \ + --member-manifest "$out/evidence/member-manifest.json" --source-report "$out/evidence/source-inspection.json" \ + --probe-log "$out/evidence/probe.log" --system-log "$out/evidence/system.log" \ + --sbom-spdx "$out/sbom/${OUTPUT_NAME%.tar.gz}.spdx.json" --sbom-cyclonedx "$out/sbom/${OUTPUT_NAME%.tar.gz}.cyclonedx.json" \ + --output "$out/evidence/payload-evidence.json" --checksums "$out/evidence/SHA256SUMS" + - name: Upload inert verified Celestia payload and evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase4-${{ matrix.family }}-linux-arm64 + path: phase4-output-${{ matrix.family }}-linux-arm64 + if-no-files-found: error + compression-level: 0 + retention-days: 30 + + toolkit-macos-arm64-build: + name: Toolkit macos/arm64 clean build ${{ matrix.build_id }} + strategy: + fail-fast: false + matrix: + build_id: [1, 2] + runs-on: macos-15 + timeout-minutes: 210 + steps: + - name: Check out exact forge source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Check out exact Midnight source in an independent tree + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + repository: midnightntwrk/midnight-node + ref: 651e043b61ed445bf7a5066c60c87ea7bd606073 + path: upstream + persist-credentials: false + fetch-depth: 1 + - name: Native locked build, inspection, probes, package, and evidence + env: + SOURCE_DATE_EPOCH: '1783616457' + MACOSX_DEPLOYMENT_TARGET: '13.0' + CARGO_INCREMENTAL: '0' + LC_ALL: C + TZ: UTC + run: | + set -euo pipefail + python3 scripts/check_runner_capability.py --runner-label macos-15 --expected-os macos --expected-arch arm64 --require-tool git --min-free-gib 10 + test "$(git -C upstream rev-parse HEAD)" = 651e043b61ed445bf7a5066c60c87ea7bd606073 + test "$(git -C upstream rev-parse 'HEAD^{tree}')" = 5c34f67538f20811d876f6463cf9aca5a3bc4fc9 + test "$(shasum -a 256 upstream/Cargo.lock | cut -d' ' -f1)" = 5e95c05593b49eda1cd017e1ebbfafa8df321fb42c9a6011f3fba3e8491fed91 + test "$SOURCE_DATE_EPOCH" = "$(git -C upstream show -s --format=%ct HEAD)" + test "$(jq -r '.node.toolkitSource.sourceDateEpoch' evidence/phase4/source-pins.json)" = "$SOURCE_DATE_EPOCH" + test "$(jq -r '.node.toolkitSource.sourceDateEpochDerivation' evidence/phase4/source-pins.json)" = "git-commit-committer-unix-seconds:$(git -C upstream rev-parse HEAD)" + test "$(jq -r '.source.buildFlags[]' catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json | grep -Fxc "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH")" = 1 + python3 scripts/validate_phase4_contract.py --root . + grep -Fq 'default = []' upstream/util/toolkit/Cargo.toml + grep -Fq 'erase-proof' upstream/util/toolkit/Cargo.toml + build_id='${{ matrix.build_id }}' + case "$build_id" in 1|2) ;; *) exit 2 ;; esac + work="$RUNNER_TEMP/phase4-macos-build" + out="$GITHUB_WORKSPACE/phase4-output-toolkit-macos-arm64-build$build_id" + mkdir -p "$work/download" "$work/tool" "$work/package" "$out/payloads" "$out/evidence" "$out/sbom" + python3 scripts/fetch_verified.py --url https://static.rust-lang.org/dist/channel-rust-1.95.0.toml \ + --output "$work/download/channel-rust-1.95.0.toml" --size 848342 \ + --sha256 821ff14e4c4a1cbe1e8915f35aff0a3fbbdf8d293ad48ab8f31e3b0440c581f9 + python3 scripts/fetch_verified.py \ + --url https://github.com/rust-secure-code/cargo-auditable/releases/download/v0.7.5/cargo-auditable-aarch64-apple-darwin.tar.xz \ + --output "$work/download/cargo-auditable-aarch64-apple-darwin.tar.xz" --size 384452 \ + --sha256 92720fff2be27492ca7b216510a27e88a835b526e9e94e46bc36b502ba17d8f0 + cargo_auditable_members=$(tar -tJf "$work/download/cargo-auditable-aarch64-apple-darwin.tar.xz") + test "$(printf '%s\n' "$cargo_auditable_members" | wc -l | tr -d ' ')" = 6 + printf '%s\n' "$cargo_auditable_members" | grep -Fxq cargo-auditable-aarch64-apple-darwin/cargo-auditable + if printf '%s\n' "$cargo_auditable_members" | grep -Eq '(^/|(^|/)\.\.(/|$)|\\)'; then exit 2; fi + tar -xJf "$work/download/cargo-auditable-aarch64-apple-darwin.tar.xz" \ + --no-same-owner --no-same-permissions --strip-components 1 -C "$work/tool" \ + cargo-auditable-aarch64-apple-darwin/cargo-auditable + chmod 0755 "$work/tool/cargo-auditable" + export PATH="$work/tool:$PATH" + rustup toolchain install 1.95.0 --profile minimal + rustup override set 1.95.0 --path upstream + { + echo 'buildCommand=cargo auditable rustc --locked --release --no-default-features -p midnight-node-toolkit --bin midnight-node-toolkit -- -C link-arg=-Wl,-no_uuid' + echo 'features=no-default-features; erase-proof forbidden' + printf 'gitCommit='; git -C upstream rev-parse HEAD + printf 'gitTree='; git -C upstream rev-parse 'HEAD^{tree}' + printf 'cargoLockSha256='; shasum -a 256 upstream/Cargo.lock | cut -d' ' -f1 + rustc +1.95.0 -Vv + cargo +1.95.0 -V + cargo auditable --version + printf 'cargoAuditablePath=%s\n' "$(command -v cargo-auditable)" + sw_vers + uname -a + uname -m + xcodebuild -version + xcrun --sdk macosx --show-sdk-path + xcrun --sdk macosx --show-sdk-version + xcrun clang --version + printf 'MACOSX_DEPLOYMENT_TARGET=%s\nSOURCE_DATE_EPOCH=%s\nCARGO_INCREMENTAL=%s\nRUNNER_TEMP=%s\n' "$MACOSX_DEPLOYMENT_TARGET" "$SOURCE_DATE_EPOCH" "$CARGO_INCREMENTAL" "$RUNNER_TEMP" + } >"$out/evidence/build-and-system.log" 2>&1 + (cd upstream && cargo auditable rustc --locked --release --no-default-features -p midnight-node-toolkit --bin midnight-node-toolkit -- -C link-arg=-Wl,-no_uuid) 2>&1 | tee -a "$out/evidence/build-and-system.log" + cp upstream/target/release/midnight-node-toolkit "$work/package/midnight-node-toolkit" + chmod 0755 "$work/package/midnight-node-toolkit" + python3 scripts/validate_native.py --binary "$work/package/midnight-node-toolkit" --os macos --arch arm64 --runner-os macos --runner-arch arm64 \ + --forbid-linkage-prefix /nix/store --forbid-linkage-prefix /opt/homebrew --forbid-linkage-prefix /usr/local + { + file "$work/package/midnight-node-toolkit" + lipo -archs "$work/package/midnight-node-toolkit" + otool -L "$work/package/midnight-node-toolkit" + vtool -show-build "$work/package/midnight-node-toolkit" + } >>"$out/evidence/build-and-system.log" + vtool -show-build "$work/package/midnight-node-toolkit" | grep -Eq 'minos[[:space:]]+13(\.0)?' + if otool -l "$work/package/midnight-node-toolkit" | grep -Fq 'cmd LC_UUID'; then exit 2; fi + if otool -L "$work/package/midnight-node-toolkit" | grep -E '/nix/store|/opt/homebrew|/usr/local'; then exit 2; fi + python3 scripts/inspect_macos_signature.py --binary "$work/package/midnight-node-toolkit" --output "$out/evidence/macos-signature.json" + bash scripts/probe_phase4_toolkit.sh "$work/package/midnight-node-toolkit" "$work/probes" + for probe in "$work/probes"/*; do printf '\n== %s ==\n' "$(basename "$probe")"; cat "$probe"; done >"$out/evidence/probe.log" + python3 scripts/manifest_single_file.py --binary "$work/package/midnight-node-toolkit" --member-name midnight-node-toolkit \ + --member-manifest "$out/evidence/member-manifest.json" --report "$out/evidence/native-build-report.json" + python3 scripts/package_deterministic.py --input-dir "$work/package" --members "$out/evidence/member-manifest.json" \ + --output "$out/payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip" + jq '{schemaVersion:"archive-policy-v1",container:"zip",maxCompressedBytes:134217728,maxExpandedBytes:268435456,maxMembers:1,maxExpansionRatio:8,expectedMembers:.members}' \ + "$out/evidence/member-manifest.json" >"$work/archive-policy.json" + python3 scripts/validate_archive.py --archive "$out/payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip" \ + --policy "$work/archive-policy.json" --scratch-parent "$RUNNER_TEMP" + bash scripts/phase4_sbom.sh "$work/package/midnight-node-toolkit" "$out/sbom/midnight-node-toolkit-macos-arm64-2.0.0-rc.4" macos arm64 + python3 scripts/emit_phase4_evidence.py --component catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json \ + --payload "$out/payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip" \ + --member-manifest "$out/evidence/member-manifest.json" --source-report "$out/evidence/native-build-report.json" \ + --probe-log "$out/evidence/probe.log" --system-log "$out/evidence/build-and-system.log" \ + --sbom-spdx "$out/sbom/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.spdx.json" \ + --sbom-cyclonedx "$out/sbom/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.cyclonedx.json" \ + --signature "$out/evidence/macos-signature.json" \ + --output "$out/evidence/payload-evidence.json" --checksums "$out/evidence/SHA256SUMS" + jq -e --arg epoch "$SOURCE_DATE_EPOCH" '.sourceDateEpoch == $epoch' "$out/evidence/native-build-report.json" + jq -e --arg epoch "$SOURCE_DATE_EPOCH" '.source.buildFlags | index("SOURCE_DATE_EPOCH=" + $epoch) != null' "$out/evidence/payload-evidence.json" + python3 scripts/validate_phase4_contract.py --root . \ + --native-report "$out/evidence/native-build-report.json" --payload-evidence "$out/evidence/payload-evidence.json" + - name: Upload inert native build payload and evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase4-toolkit-macos-arm64-build${{ matrix.build_id }} + path: phase4-output-toolkit-macos-arm64-build${{ matrix.build_id }} + if-no-files-found: error + compression-level: 0 + retention-days: 30 + + toolkit-macos-arm64-compare: + name: Toolkit macos/arm64 independent-build and clean-host gate + needs: toolkit-macos-arm64-build + runs-on: macos-15 + timeout-minutes: 45 + steps: + - name: Check out exact forge source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Download independent build 1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: phase4-toolkit-macos-arm64-build1 + path: compare/build1 + - name: Download independent build 2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: phase4-toolkit-macos-arm64-build2 + path: compare/build2 + - name: Compare exact bytes and execute with Nix and Homebrew absent + run: | + set -euo pipefail + payload=midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip + test "$(shasum -a 256 "compare/build1/payloads/$payload" | cut -d' ' -f1)" = "$(shasum -a 256 "compare/build2/payloads/$payload" | cut -d' ' -f1)" + mkdir -p "$RUNNER_TEMP/phase4-compare/build1" "$RUNNER_TEMP/phase4-compare/build2" "$RUNNER_TEMP/phase4-clean-host" + ditto -x -k "compare/build1/payloads/$payload" "$RUNNER_TEMP/phase4-compare/build1" + ditto -x -k "compare/build2/payloads/$payload" "$RUNNER_TEMP/phase4-compare/build2" + test "$(shasum -a 256 "$RUNNER_TEMP/phase4-compare/build1/midnight-node-toolkit" | cut -d' ' -f1)" = "$(shasum -a 256 "$RUNNER_TEMP/phase4-compare/build2/midnight-node-toolkit" | cut -d' ' -f1)" + ditto -x -k "compare/build1/payloads/$payload" "$RUNNER_TEMP/phase4-clean-host" + clean_bin="$RUNNER_TEMP/phase4-clean-host/midnight-node-toolkit" + test "$(lipo -archs "$clean_bin")" = arm64 + test ! -e /nix/store + hidden_brew= + restore_brew() { + if [[ -n "$hidden_brew" && -e "$hidden_brew" ]]; then sudo mv "$hidden_brew" /opt/homebrew; fi + } + trap restore_brew EXIT + if [[ -e /opt/homebrew ]]; then + hidden_brew=/opt/homebrew.phase4-hidden + test ! -e "$hidden_brew" + sudo mv /opt/homebrew "$hidden_brew" + fi + test ! -e /opt/homebrew + clean_home="$RUNNER_TEMP/phase4-clean-home" + mkdir -p "$clean_home" + PATH=/usr/bin:/bin:/usr/sbin:/sbin HOME="$clean_home" LC_ALL=C TZ=UTC \ + bash scripts/probe_phase4_toolkit.sh "$clean_bin" "$RUNNER_TEMP/phase4-clean-probes" + restore_brew + hidden_brew= + { + printf 'zipSha256=%s\n' "$(shasum -a 256 "compare/build1/payloads/$payload" | cut -d' ' -f1)" + printf 'binarySha256=%s\n' "$(shasum -a 256 "$clean_bin" | cut -d' ' -f1)" + echo 'independentBuildDigestMatch=true' + echo 'cleanRuntimeNixAbsent=true' + echo 'cleanRuntimeHomebrewAbsent=true' + echo 'cleanRuntimeProbe=PASS' + sw_vers + uname -m + otool -L "$clean_bin" + } >"$RUNNER_TEMP/phase4-independent-build-and-clean-host.log" + python3 scripts/consolidate_phase4_macos.py assemble \ + --build1 compare/build1 --build2 compare/build2 \ + --comparison-log "$RUNNER_TEMP/phase4-independent-build-and-clean-host.log" \ + --output phase4-output-toolkit-macos-arm64 + python3 scripts/consolidate_phase4_macos.py verify --root phase4-output-toolkit-macos-arm64 + - name: Upload consolidated reproducible macOS payload and evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase4-toolkit-macos-arm64 + path: phase4-output-toolkit-macos-arm64 + if-no-files-found: error + compression-level: 0 + retention-days: 30 diff --git a/catalog/components/celestia-appd-6.4.10-linux-arm64.json b/catalog/components/celestia-appd-6.4.10-linux-arm64.json new file mode 100644 index 0000000..08c2e7b --- /dev/null +++ b/catalog/components/celestia-appd-6.4.10-linux-arm64.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": "component-v1", + "componentId": "celestia-appd-6.4.10-linux-arm64", + "artifactKind": "software", + "family": "celestia-appd", + "version": "6.4.10", + "operation": "rename-only", + "source": { + "repository": "celestiaorg/celestia-app", + "commitSha": "0b26ea0cf0b52406d6004cef4a36e13c30ddca85", + "treeSha": "2fd5a31e9cb4ff326b22a1a5156003311cbcf466", + "tag": "v6.4.10", + "asset": {"id": 349955149, "name": "celestia-app_Linux_arm64.tar.gz", "url": "https://github.com/celestiaorg/celestia-app/releases/download/v6.4.10/celestia-app_Linux_arm64.tar.gz", "size": 180685852, "sha256": "52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b"} + }, + "destination": {"repository": "effectstream/binaries", "tag": "0.3.120"}, + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": {"spdx": "Apache-2.0", "evidence": [{"url": "https://raw.githubusercontent.com/celestiaorg/celestia-app/0b26ea0cf0b52406d6004cef4a36e13c30ddca85/LICENSE", "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"}]}, + "naming": {"outerTemplate": "celestia-appd-{os}-{arch}-v{version}.tar.gz", "container": "tar.gz", "appendOnly": true, "members": [{"path": "LICENSE", "type": "file", "mode": "0644"}, {"path": "README.md", "type": "file", "mode": "0644"}, {"path": "celestia-appd", "type": "file", "mode": "0755"}], "limits": {"maxCompressedBytes": 268435456, "maxExpandedBytes": 536870912, "maxMembers": 3, "maxExpansionRatio": 4}}, + "install": {"mode": "0755", "pathTemplate": "celestia-appd"}, + "targets": [{"os": "linux", "arch": "arm64", "tier": "desired", "runner": "ubuntu-24.04-arm", "native": true}], + "compatibility": {"kind": "software-runtime", "runtime": "Celestia appd 6.4.10"}, + "signing": {"applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE"}, + "sbom": {"formats": ["spdx-json", "cyclonedx-json"]}, + "validation": {"probes": ["sha256", "size", "archive-safety", "member-contract", "native-architecture", "native-linkage", "version", "help", "runtime-smoke"]} +} diff --git a/catalog/components/celestia-node-0.28.4-linux-arm64.json b/catalog/components/celestia-node-0.28.4-linux-arm64.json new file mode 100644 index 0000000..858943d --- /dev/null +++ b/catalog/components/celestia-node-0.28.4-linux-arm64.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": "component-v1", + "componentId": "celestia-node-0.28.4-linux-arm64", + "artifactKind": "software", + "family": "celestia-node", + "version": "0.28.4", + "operation": "rename-only", + "source": { + "repository": "celestiaorg/celestia-node", + "commitSha": "52cd8b52ec031bb9e3b4e476e0b159db2053384c", + "treeSha": "1f23daad3fe378c3ee51593d0c2e1d8a26f64515", + "tag": "v0.28.4", + "asset": {"id": 321528010, "name": "celestia-node_Linux_arm64.tar.gz", "url": "https://github.com/celestiaorg/celestia-node/releases/download/v0.28.4/celestia-node_Linux_arm64.tar.gz", "size": 71184641, "sha256": "09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c"} + }, + "destination": {"repository": "effectstream/binaries", "tag": "0.3.120"}, + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": {"spdx": "Apache-2.0", "evidence": [{"url": "https://raw.githubusercontent.com/celestiaorg/celestia-node/52cd8b52ec031bb9e3b4e476e0b159db2053384c/LICENSE", "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"}]}, + "naming": {"outerTemplate": "celestia-node-{os}-{arch}-v{version}.tar.gz", "container": "tar.gz", "appendOnly": true, "members": [{"path": "LICENSE", "type": "file", "mode": "0644"}, {"path": "README.md", "type": "file", "mode": "0644"}, {"path": "celestia", "type": "file", "mode": "0755"}], "limits": {"maxCompressedBytes": 134217728, "maxExpandedBytes": 268435456, "maxMembers": 3, "maxExpansionRatio": 4}}, + "install": {"mode": "0755", "pathTemplate": "celestia"}, + "targets": [{"os": "linux", "arch": "arm64", "tier": "desired", "runner": "ubuntu-24.04-arm", "native": true}], + "compatibility": {"kind": "software-runtime", "runtime": "Celestia node 0.28.4"}, + "signing": {"applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE"}, + "sbom": {"formats": ["spdx-json", "cyclonedx-json"]}, + "validation": {"probes": ["sha256", "size", "archive-safety", "member-contract", "native-architecture", "native-linkage", "version", "help", "runtime-smoke"]} +} diff --git a/catalog/components/midnight-node-2.0.0-rc.4-linux-arm64.json b/catalog/components/midnight-node-2.0.0-rc.4-linux-arm64.json new file mode 100644 index 0000000..9a32172 --- /dev/null +++ b/catalog/components/midnight-node-2.0.0-rc.4-linux-arm64.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": "component-v1", + "componentId": "midnight-node-2.0.0-rc.4-linux-arm64", + "artifactKind": "software", + "family": "midnight-node", + "version": "2.0.0-rc.4", + "operation": "repackage", + "source": { + "repository": "midnightntwrk/midnight-node", + "commitSha": "651e043b61ed445bf7a5066c60c87ea7bd606073", + "treeSha": "5c34f67538f20811d876f6463cf9aca5a3bc4fc9", + "tag": "node-2.0.0-rc.4", + "asset": { + "id": 471563200, + "name": "midnight-node-2.0.0-rc.4-linux-arm64.tar.gz", + "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/midnight-node-2.0.0-rc.4-linux-arm64.tar.gz", + "size": 83084349, + "sha256": "2bfd0f768a88d0d1d49cc97bfd6b9f4401646458887219e8c021bff8d9a9ae0b" + }, + "lockedDependencies": true, + "toolchain": "forge-phase4-transformation-toolchain-v1@sha256:141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e", + "toolchainDigest": "141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e", + "buildFlags": ["verified-tar-gz-to-deterministic-zip", "rename-root-executable-to-versioned-family-name", "preserve-exact-res-tree", "zip-deflate-9", "zip-epoch-1980-01-01T00:00:00Z"] + }, + "destination": {"repository": "effectstream/binaries", "tag": "0.3.120"}, + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": { + "spdx": "Apache-2.0", + "evidence": [{"url": "https://raw.githubusercontent.com/midnightntwrk/midnight-node/651e043b61ed445bf7a5066c60c87ea7bd606073/LICENSE", "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"}], + "redistributionStatement": "Apache-2.0 source license at the exact release commit; official release asset is transformed only into the established warehouse ZIP layout." + }, + "naming": { + "outerTemplate": "midnight-node-{os}-{arch}-{version}.zip", + "container": "zip", + "appendOnly": true, + "members": [ + {"path": "midnight-node-linux-arm64-2.0.0-rc.4", "type": "file", "mode": "0755"}, + {"path": "res", "type": "directory", "mode": "0755"} + ], + "limits": {"maxCompressedBytes": 134217728, "maxExpandedBytes": 536870912, "maxMembers": 512, "maxExpansionRatio": 8} + }, + "install": {"mode": "0755", "pathTemplate": "midnight-node-linux-arm64-2.0.0-rc.4"}, + "targets": [{"os": "linux", "arch": "arm64", "tier": "desired", "runner": "ubuntu-24.04-arm", "native": true}], + "compatibility": {"kind": "software-runtime", "runtime": "Midnight node release identity 2.0.0-rc.4; resource tree is mandatory", "probes": ["version", "help", "start-with-res", "missing-res-rejection"]}, + "signing": {"applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE"}, + "sbom": {"formats": ["spdx-json", "cyclonedx-json"]}, + "validation": {"probes": ["sha256", "size", "archive-safety", "member-contract", "native-architecture", "native-linkage", "version", "help", "runtime-smoke"]} +} diff --git a/catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json b/catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json new file mode 100644 index 0000000..e42b27b --- /dev/null +++ b/catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": "component-v1", + "componentId": "midnight-node-toolkit-2.0.0-rc.4-linux-amd64", + "artifactKind": "software", + "family": "midnight-node-toolkit", + "version": "2.0.0-rc.4", + "operation": "repackage", + "source": { + "repository": "midnightntwrk/midnight-node", + "commitSha": "651e043b61ed445bf7a5066c60c87ea7bd606073", + "treeSha": "5c34f67538f20811d876f6463cf9aca5a3bc4fc9", + "tag": "node-2.0.0-rc.4", + "asset": {"id": 471563135, "name": "midnight-node-toolkit-2.0.0-rc.4-linux-amd64.tar.gz", "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.tar.gz", "size": 50581636, "sha256": "fdd9f6974c1c20928d3b9dc79c13c51bf984eb6dc4cdaf1978a1b037794402de"}, + "lockedDependencies": true, + "toolchain": "forge-phase4-transformation-toolchain-v1@sha256:141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e", + "toolchainDigest": "141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e", + "buildFlags": ["official-binary-not-rebuilt", "single-root-executable", "zip-deflate-9", "zip-epoch-1980-01-01T00:00:00Z"] + }, + "destination": {"repository": "effectstream/binaries", "tag": "0.3.120"}, + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": {"spdx": "Apache-2.0", "evidence": [{"url": "https://raw.githubusercontent.com/midnightntwrk/midnight-node/651e043b61ed445bf7a5066c60c87ea7bd606073/LICENSE", "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"}]}, + "naming": {"outerTemplate": "midnight-node-toolkit-{os}-{arch}-{version}.zip", "container": "zip", "appendOnly": true, "members": [{"path": "midnight-node-toolkit", "type": "file", "mode": "0755"}], "limits": {"maxCompressedBytes": 100663296, "maxExpandedBytes": 268435456, "maxMembers": 1, "maxExpansionRatio": 8}}, + "install": {"mode": "0755", "pathTemplate": "midnight-node-toolkit"}, + "targets": [{"os": "linux", "arch": "amd64", "tier": "required", "runner": "ubuntu-24.04", "native": true}], + "compatibility": {"kind": "software-runtime", "runtime": "Midnight node 2.0.0 / base Ledger 7.0.3 / CompactC 0.31.0-6587676a9bb2; release identity 2.0.0-rc.4"}, + "signing": {"applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE"}, + "sbom": {"formats": ["spdx-json", "cyclonedx-json"]}, + "validation": {"probes": ["sha256", "size", "archive-safety", "member-contract", "native-architecture", "native-linkage", "version", "help", "runtime-smoke"]} +} diff --git a/catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json b/catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json new file mode 100644 index 0000000..bade69b --- /dev/null +++ b/catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": "component-v1", + "componentId": "midnight-node-toolkit-2.0.0-rc.4-linux-arm64", + "artifactKind": "software", + "family": "midnight-node-toolkit", + "version": "2.0.0-rc.4", + "operation": "repackage", + "source": { + "repository": "midnightntwrk/midnight-node", + "commitSha": "651e043b61ed445bf7a5066c60c87ea7bd606073", + "treeSha": "5c34f67538f20811d876f6463cf9aca5a3bc4fc9", + "tag": "node-2.0.0-rc.4", + "asset": {"id": 471563203, "name": "midnight-node-toolkit-2.0.0-rc.4-linux-arm64.tar.gz", "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.tar.gz", "size": 48937006, "sha256": "85513afbfe4dd3fccadde6ad5d7ccf82b8fa0c88d010445fc62c223c9ef078eb"}, + "lockedDependencies": true, + "toolchain": "forge-phase4-transformation-toolchain-v1@sha256:141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e", + "toolchainDigest": "141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e", + "buildFlags": ["official-binary-not-rebuilt", "single-root-executable", "zip-deflate-9", "zip-epoch-1980-01-01T00:00:00Z"] + }, + "destination": {"repository": "effectstream/binaries", "tag": "0.3.120"}, + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": {"spdx": "Apache-2.0", "evidence": [{"url": "https://raw.githubusercontent.com/midnightntwrk/midnight-node/651e043b61ed445bf7a5066c60c87ea7bd606073/LICENSE", "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"}]}, + "naming": {"outerTemplate": "midnight-node-toolkit-{os}-{arch}-{version}.zip", "container": "zip", "appendOnly": true, "members": [{"path": "midnight-node-toolkit", "type": "file", "mode": "0755"}], "limits": {"maxCompressedBytes": 100663296, "maxExpandedBytes": 268435456, "maxMembers": 1, "maxExpansionRatio": 8}}, + "install": {"mode": "0755", "pathTemplate": "midnight-node-toolkit"}, + "targets": [{"os": "linux", "arch": "arm64", "tier": "desired", "runner": "ubuntu-24.04-arm", "native": true}], + "compatibility": {"kind": "software-runtime", "runtime": "Midnight node 2.0.0 / base Ledger 7.0.3 / CompactC 0.31.0-6587676a9bb2; release identity 2.0.0-rc.4"}, + "signing": {"applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE"}, + "sbom": {"formats": ["spdx-json", "cyclonedx-json"]}, + "validation": {"probes": ["sha256", "size", "archive-safety", "member-contract", "native-architecture", "native-linkage", "version", "help", "runtime-smoke"]} +} diff --git a/catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json b/catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json new file mode 100644 index 0000000..0265e92 --- /dev/null +++ b/catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": "component-v1", + "componentId": "midnight-node-toolkit-2.0.0-rc.4-macos-arm64", + "artifactKind": "software", + "family": "midnight-node-toolkit", + "version": "2.0.0-rc.4", + "operation": "build", + "source": { + "repository": "midnightntwrk/midnight-node", + "commitSha": "651e043b61ed445bf7a5066c60c87ea7bd606073", + "treeSha": "5c34f67538f20811d876f6463cf9aca5a3bc4fc9", + "tag": "node-2.0.0-rc.4", + "lockedDependencies": true, + "toolchain": "rustup-channel-1.95.0@sha256:821ff14e4c4a1cbe1e8915f35aff0a3fbbdf8d293ad48ab8f31e3b0440c581f9", + "toolchainDigest": "821ff14e4c4a1cbe1e8915f35aff0a3fbbdf8d293ad48ab8f31e3b0440c581f9", + "buildFlags": ["cargo-auditable-0.7.5-asset-sha256:92720fff2be27492ca7b216510a27e88a835b526e9e94e46bc36b502ba17d8f0", "cargo-auditable-wrapper-path:$RUNNER_TEMP/phase4-macos-build/tool/cargo-auditable", "forge-phase4-evidence-closure-toolchain-v1@sha256:a5f6a0b22ec5e511d8b85641edb94226c89f4471fa417e752609112f242bfcc5", "cargo auditable rustc --locked --release --no-default-features -p midnight-node-toolkit --bin midnight-node-toolkit -- -C link-arg=-Wl,-no_uuid", "no-default-features", "forbid-erase-proof", "MACOSX_DEPLOYMENT_TARGET=13.0", "SOURCE_DATE_EPOCH=1783616457", "CARGO_INCREMENTAL=0"] + }, + "destination": {"repository": "effectstream/binaries", "tag": "0.3.120"}, + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": {"spdx": "Apache-2.0", "evidence": [{"url": "https://raw.githubusercontent.com/midnightntwrk/midnight-node/651e043b61ed445bf7a5066c60c87ea7bd606073/LICENSE", "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"}]}, + "naming": {"outerTemplate": "midnight-node-toolkit-{os}-{arch}-{version}.zip", "container": "zip", "appendOnly": true, "members": [{"path": "midnight-node-toolkit", "type": "file", "mode": "0755"}], "limits": {"maxCompressedBytes": 134217728, "maxExpandedBytes": 268435456, "maxMembers": 1, "maxExpansionRatio": 8}}, + "install": {"mode": "0755", "pathTemplate": "midnight-node-toolkit"}, + "targets": [{"os": "macos", "arch": "arm64", "tier": "required", "runner": "macos-15", "native": true}], + "compatibility": {"kind": "software-runtime", "runtime": "Source package 2.0.0 / Midnight node 2.0.0 / base Ledger 7.0.3 / CompactC 0.31.0-6587676a9bb2; release identity 2.0.0-rc.4"}, + "signing": {"applicability": "macos", "distributionSigningState": "UNSIGNED_DEVELOPMENT_ONLY"}, + "sbom": {"formats": ["spdx-json", "cyclonedx-json"]}, + "validation": {"probes": ["sha256", "size", "archive-safety", "member-contract", "native-architecture", "native-linkage", "version", "help", "runtime-smoke"]} +} diff --git a/evidence/phase4/evidence-closure-toolchain.json b/evidence/phase4/evidence-closure-toolchain.json new file mode 100644 index 0000000..38ee66d --- /dev/null +++ b/evidence/phase4/evidence-closure-toolchain.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": "phase4-evidence-closure-toolchain-v1", + "scripts": [ + { + "path": "scripts/consolidate_phase4_macos.py", + "sha256": "f1aad73e6c4a66e680ca9e07a33979bd5db9aebae39b1d0aee87b1a7d3d0c6b5" + }, + { + "path": "scripts/validate_phase4_contract.py", + "sha256": "64bc44edf24db4fc6df27b835b95b72df257dd99efe077c627002fdbb05f33d3" + } + ] +} diff --git a/evidence/phase4/source-pins.json b/evidence/phase4/source-pins.json new file mode 100644 index 0000000..b05e11f --- /dev/null +++ b/evidence/phase4/source-pins.json @@ -0,0 +1,193 @@ +{ + "schemaVersion": "phase4-source-pins-v1", + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "warning": "DEVELOPMENT ONLY — NOT FOR PRODUCTION USE. Release 0.3.120 is mutable; verify every downloaded SHA-256 against committed metadata before installation or execution.", + "node": { + "repository": "midnightntwrk/midnight-node", + "repositoryId": 987173205, + "repositoryNodeId": "R_kgDOOtcRVQ", + "tag": "node-2.0.0-rc.4", + "commitSha": "651e043b61ed445bf7a5066c60c87ea7bd606073", + "treeSha": "5c34f67538f20811d876f6463cf9aca5a3bc4fc9", + "releaseId": 351676950, + "releaseNodeId": "RE_kwDOOtcRVc4U9ioW", + "license": { + "spdx": "Apache-2.0", + "url": "https://raw.githubusercontent.com/midnightntwrk/midnight-node/651e043b61ed445bf7a5066c60c87ea7bd606073/LICENSE", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + }, + "cargoLock": { + "url": "https://raw.githubusercontent.com/midnightntwrk/midnight-node/651e043b61ed445bf7a5066c60c87ea7bd606073/Cargo.lock", + "gitBlobSha": "d94267ba7457898ae1fec4317b60f1ac58c440e8", + "size": 502022, + "sha256": "5e95c05593b49eda1cd017e1ebbfafa8df321fb42c9a6011f3fba3e8491fed91" + }, + "rustToolchain": { + "channel": "1.95", + "resolved": "1.95.0", + "sourcePath": "rust-toolchain.toml", + "gitBlobSha": "098758ffb9b17caadb283317cf54fc9788b74f6a", + "channelManifestUrl": "https://static.rust-lang.org/dist/channel-rust-1.95.0.toml", + "channelManifestSha256": "821ff14e4c4a1cbe1e8915f35aff0a3fbbdf8d293ad48ab8f31e3b0440c581f9" + }, + "darwinLockedPackages": [ + "core-foundation", + "core-foundation-sys", + "libc", + "mach2", + "psm", + "security-framework", + "security-framework-sys", + "stacker", + "system-configuration", + "system-configuration-sys" + ], + "assets": { + "nodeLinuxArm64": { + "id": 471563200, + "nodeId": "RA_kwDOOtcRVc4cG3vA", + "name": "midnight-node-2.0.0-rc.4-linux-arm64.tar.gz", + "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/midnight-node-2.0.0-rc.4-linux-arm64.tar.gz", + "size": 83084349, + "sha256": "2bfd0f768a88d0d1d49cc97bfd6b9f4401646458887219e8c021bff8d9a9ae0b" + }, + "toolkitLinuxAmd64": { + "id": 471563135, + "nodeId": "RA_kwDOOtcRVc4cG3t_", + "name": "midnight-node-toolkit-2.0.0-rc.4-linux-amd64.tar.gz", + "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.tar.gz", + "size": 50581636, + "sha256": "fdd9f6974c1c20928d3b9dc79c13c51bf984eb6dc4cdaf1978a1b037794402de" + }, + "toolkitLinuxArm64": { + "id": 471563203, + "nodeId": "RA_kwDOOtcRVc4cG3vD", + "name": "midnight-node-toolkit-2.0.0-rc.4-linux-arm64.tar.gz", + "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.tar.gz", + "size": 48937006, + "sha256": "85513afbfe4dd3fccadde6ad5d7ccf82b8fa0c88d010445fc62c223c9ef078eb" + }, + "checksums": { + "id": 471581595, + "nodeId": "RA_kwDOOtcRVc4cG8Ob", + "name": "SHA256SUMS", + "url": "https://github.com/midnightntwrk/midnight-node/releases/download/node-2.0.0-rc.4/SHA256SUMS", + "size": 795, + "sha256": "f75fc8e097fb6b77113a4d92814d783e01068bf4440a19bd4dc3e83ef96bc905" + } + }, + "toolkitSource": { + "package": "midnight-node-toolkit", + "packageVersion": "2.0.0", + "releaseIdentity": "2.0.0-rc.4", + "manifestPath": "util/toolkit/Cargo.toml", + "manifestGitBlobSha": "3460851f619d7fd52f742d75e2940a873b1a4bd7", + "defaultFeatures": [], + "forbiddenFeatures": ["erase-proof"], + "buildCommand": "cargo auditable rustc --locked --release --no-default-features -p midnight-node-toolkit --bin midnight-node-toolkit -- -C link-arg=-Wl,-no_uuid", + "cargoAuditableWrapperPath": "$RUNNER_TEMP/phase4-macos-build/tool/cargo-auditable", + "macosDeploymentTarget": "13.0", + "sourceDateEpoch": "1783616457", + "sourceDateEpochDerivation": "git-commit-committer-unix-seconds:651e043b61ed445bf7a5066c60c87ea7bd606073", + "evidenceClosureToolchain": { + "path": "evidence/phase4/evidence-closure-toolchain.json", + "sha256": "a5f6a0b22ec5e511d8b85641edb94226c89f4471fa417e752609112f242bfcc5", + "locator": "forge-phase4-evidence-closure-toolchain-v1@sha256:a5f6a0b22ec5e511d8b85641edb94226c89f4471fa417e752609112f242bfcc5" + } + } + }, + "cargoAuditable": { + "version": "0.7.5", + "commitSha": "9dc5ff28ff204b73eae91bd96cde4413fdcb70e0", + "releaseId": 327406646, + "releaseNodeId": "RE_kwDOCdwzys4Tg9Q2", + "asset": { + "id": 426566625, + "nodeId": "RA_kwDOCdwzys4ZbOPh", + "name": "cargo-auditable-aarch64-apple-darwin.tar.xz", + "url": "https://github.com/rust-secure-code/cargo-auditable/releases/download/v0.7.5/cargo-auditable-aarch64-apple-darwin.tar.xz", + "size": 384452, + "sha256": "92720fff2be27492ca7b216510a27e88a835b526e9e94e46bc36b502ba17d8f0" + } + }, + "celestiaApp": { + "repository": "celestiaorg/celestia-app", + "repositoryId": 327696712, + "repositoryNodeId": "MDEwOlJlcG9zaXRvcnkzMjc2OTY3MTI=", + "tag": "v6.4.10", + "commitSha": "0b26ea0cf0b52406d6004cef4a36e13c30ddca85", + "treeSha": "2fd5a31e9cb4ff326b22a1a5156003311cbcf466", + "releaseId": 282480015, + "releaseNodeId": "RE_kwDOE4hBSM4Q1k2P", + "license": { + "spdx": "Apache-2.0", + "url": "https://raw.githubusercontent.com/celestiaorg/celestia-app/0b26ea0cf0b52406d6004cef4a36e13c30ddca85/LICENSE", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + }, + "asset": { + "id": 349955149, + "nodeId": "RA_kwDOE4hBSM4U2-RN", + "name": "celestia-app_Linux_arm64.tar.gz", + "url": "https://github.com/celestiaorg/celestia-app/releases/download/v6.4.10/celestia-app_Linux_arm64.tar.gz", + "size": 180685852, + "sha256": "52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b" + }, + "checksums": { + "id": 349955278, + "nodeId": "RA_kwDOE4hBSM4U2-TO", + "name": "checksums.txt", + "url": "https://github.com/celestiaorg/celestia-app/releases/download/v6.4.10/checksums.txt", + "size": 836, + "sha256": "960c02e97215f6c802e0992a64458b3f3df04cf2fb56f2f9e802a204a7a9e5c1" + } + }, + "celestiaNode": { + "repository": "celestiaorg/celestia-node", + "repositoryId": 396780483, + "repositoryNodeId": "MDEwOlJlcG9zaXRvcnkzOTY3ODA0ODM=", + "tag": "v0.28.4", + "commitSha": "52cd8b52ec031bb9e3b4e476e0b159db2053384c", + "treeSha": "1f23daad3fe378c3ee51593d0c2e1d8a26f64515", + "releaseId": 265784044, + "releaseNodeId": "RE_kwDOF6Zjw84P14rs", + "license": { + "spdx": "Apache-2.0", + "url": "https://raw.githubusercontent.com/celestiaorg/celestia-node/52cd8b52ec031bb9e3b4e476e0b159db2053384c/LICENSE", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + }, + "asset": { + "id": 321528010, + "nodeId": "RA_kwDOF6Zjw84TKiDK", + "name": "celestia-node_Linux_arm64.tar.gz", + "url": "https://github.com/celestiaorg/celestia-node/releases/download/v0.28.4/celestia-node_Linux_arm64.tar.gz", + "size": 71184641, + "sha256": "09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c" + }, + "checksums": { + "id": 321528021, + "nodeId": "RA_kwDOF6Zjw84TKiDV", + "name": "checksums.txt", + "url": "https://github.com/celestiaorg/celestia-node/releases/download/v0.28.4/checksums.txt", + "size": 400, + "sha256": "10b730253822bff3be9517f649d56b26ebfa192f23a40b9a0b1b2313a55425ec" + }, + "checksumSignature": { + "id": 321528024, + "nodeId": "RA_kwDOF6Zjw84TKiDY", + "name": "checksums.txt.sig", + "url": "https://github.com/celestiaorg/celestia-node/releases/download/v0.28.4/checksums.txt.sig", + "size": 119, + "sha256": "7adf892e84c42c362d867e91348f6a8686904edad37aa07fc4e41570267efa5b" + } + }, + "syft": { + "version": "1.51.1", + "releaseTag": "v1.51.1", + "assets": { + "linuxAmd64": {"id": 532610151, "name": "syft_1.51.1_linux_amd64.tar.gz", "size": 29203595, "sha256": "8fcb33017a0dc1058298c923c436d19dfa68ae93968e0b423248542e3afb9fc3"}, + "linuxArm64": {"id": 532610134, "name": "syft_1.51.1_linux_arm64.tar.gz", "size": 26668605, "sha256": "a7fd2b784e6664acd44719270574f6cd8c6864fc2b1700bf9099bd1cccda7d7f"}, + "macosArm64": {"id": 532610139, "name": "syft_1.51.1_darwin_arm64.tar.gz", "size": 27907057, "sha256": "ac063af3b9874769deb7ea1e6d76841e68f9e3bb50cd654226fc977de65532c1"} + } + } +} diff --git a/evidence/phase4/transformation-toolchain.json b/evidence/phase4/transformation-toolchain.json new file mode 100644 index 0000000..63d6be7 --- /dev/null +++ b/evidence/phase4/transformation-toolchain.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "phase4-transformation-toolchain-v1", + "python": "3.12", + "zipCompression": "deflate-9", + "scripts": [ + { + "path": "scripts/fetch_verified.py", + "sha256": "23302c027b40b74157b8fbf2d8de8be2cd5ea7cc390a54774617c32d5f3a67f2" + }, + { + "path": "scripts/package_deterministic.py", + "sha256": "4b82303be6a60a439a398a24ea9eb52bc49ed148d81b1e62a887f66870cea8bb" + }, + { + "path": "scripts/phase4_payloads.py", + "sha256": "3c5f75d2bb16fe7fa5527e88fd1c9a1cbecfc317efc8bbba206c95e10c529d7d" + } + ] +} diff --git a/scripts/consolidate_phase4_macos.py b/scripts/consolidate_phase4_macos.py new file mode 100644 index 0000000..91b9787 --- /dev/null +++ b/scripts/consolidate_phase4_macos.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Assemble and verify a complete checksum-closed two-build macOS candidate.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import shutil +import sys +from pathlib import Path, PurePosixPath + +from forge_io import ForgeError, create_file_atomic, expect, load_json, sha256_file + + +PAYLOAD = "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip" +SBOM_STEM = "midnight-node-toolkit-macos-arm64-2.0.0-rc.4" +BUILD_FILES = { + f"payloads/{PAYLOAD}", + "evidence/SHA256SUMS", + "evidence/build-and-system.log", + "evidence/macos-signature.json", + "evidence/member-manifest.json", + "evidence/native-build-report.json", + "evidence/payload-evidence.json", + "evidence/probe.log", + f"sbom/{SBOM_STEM}.cyclonedx.json", + f"sbom/{SBOM_STEM}.spdx.json", +} +CHECKSUM_RE = re.compile(r"^([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._-]{0,255})$") + + +def regular_files(root: Path) -> dict[str, Path]: + expect(root.is_dir() and not root.is_symlink(), f"unsafe or missing directory: {root}") + result: dict[str, Path] = {} + for path in sorted(root.rglob("*")): + expect(not path.is_symlink(), f"symlink forbidden in evidence tree: {path}") + if path.is_dir(): + continue + expect(path.is_file(), f"non-regular evidence object: {path}") + relative = path.relative_to(root).as_posix() + expect(relative not in result, f"duplicate evidence path: {relative}") + result[relative] = path + return result + + +def identity(path: Path) -> tuple[str, int]: + return sha256_file(path, 512 * 2**20) + + +def validate_build(root: Path, allow_comparison_log: bool = False) -> dict[str, object]: + files = regular_files(root) + expected = set(BUILD_FILES) + if allow_comparison_log: + expected.add("evidence/independent-build-and-clean-host.log") + allowed_extra = {"SHA256SUMS"} | {name for name in files if name.startswith("independent-builds/build2/")} + else: + allowed_extra = set() + expect(set(files) == expected | allowed_extra, f"build evidence file set mismatch: missing={sorted(expected - set(files))}, extra={sorted(set(files) - expected - allowed_extra)}") + build_files = {name: files[name] for name in expected} + + record = load_json(build_files["evidence/payload-evidence.json"]) + payload = record["payload"] + expect(payload["name"] == PAYLOAD, "payload evidence name mismatch") + digest, size = identity(build_files[f"payloads/{PAYLOAD}"]) + expect((payload["sha256"], payload["size"]) == (digest, size), "payload evidence identity mismatch") + + referenced = {PAYLOAD, "payload-evidence.json"} + candidates: dict[str, Path] = {} + for relative, path in build_files.items(): + if relative == "evidence/SHA256SUMS" or relative.endswith("independent-build-and-clean-host.log"): + continue + expect(path.name not in candidates, f"ambiguous evidence basename: {path.name}") + candidates[path.name] = path + for row in record["evidence"]: + name = row["name"] + expect(name in candidates, f"dangling payload evidence reference: {name}") + observed_digest, observed_size = identity(candidates[name]) + expect((row["sha256"], row["size"]) == (observed_digest, observed_size), f"payload evidence identity mismatch: {name}") + referenced.add(name) + + checksum_rows: dict[str, str] = {} + for line in build_files["evidence/SHA256SUMS"].read_text(encoding="utf-8").splitlines(): + match = CHECKSUM_RE.fullmatch(line) + expect(match is not None, f"invalid build checksum row: {line!r}") + assert match is not None + digest, name = match.groups() + expect(name not in checksum_rows, f"duplicate build checksum name: {name}") + expect(name in candidates, f"dangling build checksum name: {name}") + expect(identity(candidates[name])[0] == digest, f"build checksum mismatch: {name}") + checksum_rows[name] = digest + expect(set(checksum_rows) == referenced, f"build checksum closure mismatch: missing={sorted(referenced - set(checksum_rows))}, extra={sorted(set(checksum_rows) - referenced)}") + return record + + +def root_rows(root: Path) -> list[str]: + files = regular_files(root) + files.pop("SHA256SUMS", None) + return [f"{identity(path)[0]} {relative}\n" for relative, path in sorted(files.items())] + + +def verify(root: Path) -> None: + files = regular_files(root) + expect("SHA256SUMS" in files, "root SHA256SUMS missing") + expected_paths = set(files) - {"SHA256SUMS"} + rows: dict[str, str] = {} + previous = "" + for line in files["SHA256SUMS"].read_text(encoding="utf-8").splitlines(): + expect(" " in line, f"invalid root checksum row: {line!r}") + digest, relative = line.split(" ", 1) + expect(re.fullmatch(r"[0-9a-f]{64}", digest) is not None, f"invalid root checksum digest: {relative}") + path = PurePosixPath(relative) + expect(relative == path.as_posix() and not path.is_absolute() and all(part not in {"", ".", ".."} for part in path.parts), f"unsafe root checksum path: {relative!r}") + expect(relative > previous, "root checksum rows must be strictly sorted") + previous = relative + expect(relative not in rows, f"duplicate root checksum path: {relative}") + expect(relative in files, f"dangling root checksum path: {relative}") + expect(identity(files[relative])[0] == digest, f"root checksum mismatch: {relative}") + rows[relative] = digest + expect(set(rows) == expected_paths, f"root checksum file-set mismatch: missing={sorted(expected_paths - set(rows))}, extra={sorted(set(rows) - expected_paths)}") + + build1 = validate_build(root, allow_comparison_log=True) + build2_root = root / "independent-builds/build2" + build2 = validate_build(build2_root) + expect(build1["payload"] == build2["payload"], "independent payload evidence differs") + expect(build1["signing"] == build2["signing"], "independent signature evidence differs") + expect(load_json(root / "evidence/native-build-report.json") == load_json(build2_root / "evidence/native-build-report.json"), "independent native build report differs") + + +def assemble(build1: Path, build2: Path, comparison_log: Path, output: Path) -> None: + record1 = validate_build(build1) + record2 = validate_build(build2) + expect(record1["payload"] == record2["payload"], "independent payload evidence differs") + expect(record1["signing"] == record2["signing"], "independent signature evidence differs") + expect(comparison_log.is_file() and not comparison_log.is_symlink(), "comparison log missing or unsafe") + expect(not output.exists(), f"output already exists: {output}") + shutil.copytree(build1, output, symlinks=False) + shutil.copytree(build2, output / "independent-builds/build2", symlinks=False) + shutil.copy2(comparison_log, output / "evidence/independent-build-and-clean-host.log") + create_file_atomic(output / "SHA256SUMS", "".join(root_rows(output)).encode("utf-8")) + verify(output) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + assemble_parser = subparsers.add_parser("assemble") + assemble_parser.add_argument("--build1", required=True, type=Path) + assemble_parser.add_argument("--build2", required=True, type=Path) + assemble_parser.add_argument("--comparison-log", required=True, type=Path) + assemble_parser.add_argument("--output", required=True, type=Path) + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("--root", required=True, type=Path) + args = parser.parse_args() + try: + if args.command == "assemble": + assemble(args.build1, args.build2, args.comparison_log, args.output) + else: + verify(args.root) + print(f"OK Phase-4 macOS complete evidence closure ({args.command})") + return 0 + except (ForgeError, OSError, KeyError, TypeError, ValueError, shutil.Error) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/emit_phase4_evidence.py b/scripts/emit_phase4_evidence.py new file mode 100755 index 0000000..ab93769 --- /dev/null +++ b/scripts/emit_phase4_evidence.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Emit a digest-bound Phase-4 payload/evidence record.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, load_json, sha256_file + + +def identity(path: Path) -> dict[str, object]: + digest, size = sha256_file(path, 2 * 2**30) + return {"name": path.name, "size": size, "sha256": digest} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--component", required=True, type=Path) + parser.add_argument("--payload", required=True, type=Path) + parser.add_argument("--member-manifest", required=True, type=Path) + parser.add_argument("--source-report", required=True, type=Path) + parser.add_argument("--probe-log", required=True, type=Path) + parser.add_argument("--system-log", required=True, type=Path) + parser.add_argument("--sbom-spdx", required=True, type=Path) + parser.add_argument("--sbom-cyclonedx", required=True, type=Path) + parser.add_argument("--signature", type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--checksums", required=True, type=Path) + args = parser.parse_args() + try: + component = load_json(args.component) + paths = [ + args.payload, + args.member_manifest, + args.source_report, + args.probe_log, + args.system_log, + args.sbom_spdx, + args.sbom_cyclonedx, + ] + if args.signature is not None: + paths.append(args.signature) + expect(all(path.is_file() and not path.is_symlink() for path in paths), "Phase-4 evidence input missing or unsafe") + evidence = [identity(path) for path in paths if path != args.payload] + record = { + "schemaVersion": "phase4-payload-evidence-v1", + "componentId": component["componentId"], + "artifactKind": "software", + "payload": identity(args.payload), + "source": component["source"], + "destination": component["destination"], + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "license": component["license"], + "targets": component["targets"], + "compatibility": component["compatibility"], + "signing": load_json(args.signature) if args.signature is not None else component["signing"], + "evidence": sorted(evidence, key=lambda row: str(row["name"])), + "builder": { + "repository": os.environ.get("GITHUB_REPOSITORY", "local-test"), + "sourceSha": os.environ.get("GITHUB_SHA", "0" * 40), + "workflow": os.environ.get("GITHUB_WORKFLOW", "local-test"), + "runId": os.environ.get("GITHUB_RUN_ID", "local-test"), + "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT", "local-test"), + "runnerOs": os.environ.get("RUNNER_OS", sys.platform), + "runnerArch": os.environ.get("RUNNER_ARCH", "unknown"), + "runnerImageOs": os.environ.get("ImageOS"), + "runnerImageVersion": os.environ.get("ImageVersion"), + }, + } + create_file_atomic(args.output, canonical_bytes(record) + b"\n") + checksum_paths = paths + [args.output] + rows = [] + for path in sorted(checksum_paths, key=lambda item: item.name): + digest, _ = sha256_file(path, 2 * 2**30) + rows.append(f"{digest} {path.name}\n") + create_file_atomic(args.checksums, "".join(rows).encode("utf-8")) + print(json.dumps(record, sort_keys=True, separators=(",", ":"))) + return 0 + except (ForgeError, OSError, KeyError, TypeError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/inspect_macos_signature.py b/scripts/inspect_macos_signature.py new file mode 100755 index 0000000..ce2c6e6 --- /dev/null +++ b/scripts/inspect_macos_signature.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Capture the actual pre-Developer-ID signature state of one Mach-O file.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, sha256_file + + +def run(command: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=30, check=False) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + digest, size = sha256_file(args.binary, 512 * 2**20) + display = run(["codesign", "--display", "--verbose=4", str(args.binary)]) + combined = display.stdout + display.stderr + if display.returncode != 0: + expect("not signed" in combined.casefold(), f"unexpected codesign display failure: {combined.strip()}") + kind = "none" + strict = False + cdhash = None + authorities: list[str] = [] + team_id = None + hardened = False + else: + authorities = re.findall(r"^Authority=(.+)$", combined, flags=re.MULTILINE) + team_match = re.search(r"^TeamIdentifier=(.+)$", combined, flags=re.MULTILINE) + team_id = None if team_match is None or team_match.group(1) in {"not set", ""} else team_match.group(1) + cdhash_match = re.search(r"^CDHash=([0-9A-Fa-f]+)$", combined, flags=re.MULTILINE) + expect(cdhash_match is not None, "signed Mach-O has no CDHash") + cdhash = cdhash_match.group(1).lower() + adhoc = bool(re.search(r"^Signature=adhoc$", combined, flags=re.MULTILINE)) + kind = "linker-adhoc" if adhoc and not authorities and team_id is None else "developer-id" + verify = run(["codesign", "--verify", "--strict", "--verbose=4", str(args.binary)]) + strict = verify.returncode == 0 + hardened = "runtime" in combined.casefold() and "flags=" in combined.casefold() + expect(kind in {"none", "linker-adhoc"}, "initial development build unexpectedly carries Developer ID authority") + record = { + "schemaVersion": "phase4-macos-signature-v1", + "distributionSigningState": "UNSIGNED_DEVELOPMENT_ONLY", + "binary": {"name": args.binary.name, "size": size, "sha256": digest}, + "codeSignatureKind": kind, + "cdHash": cdhash, + "authorities": authorities, + "teamId": team_id, + "hardenedRuntime": hardened, + "strictVerification": strict, + "codesignDisplayExit": display.returncode, + "codesignDisplay": combined, + } + create_file_atomic(args.output, canonical_bytes(record) + b"\n") + print(json.dumps(record, sort_keys=True, separators=(",", ":"))) + return 0 + except (ForgeError, OSError, subprocess.SubprocessError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/manifest_single_file.py b/scripts/manifest_single_file.py new file mode 100755 index 0000000..e1d8999 --- /dev/null +++ b/scripts/manifest_single_file.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Emit an exact one-root-executable member manifest and build report.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, safe_basename, sha256_file, validate_regular_file + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True, type=Path) + parser.add_argument("--member-name", required=True) + parser.add_argument("--member-manifest", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + args = parser.parse_args() + try: + name = safe_basename(args.member_name, "member name") + validate_regular_file(args.binary, "0755") + digest, size = sha256_file(args.binary, 512 * 2**20) + manifest = {"schemaVersion": "member-manifest-v1", "members": [{"path": name, "type": "file", "mode": "0755", "size": size, "sha256": digest}]} + create_file_atomic(args.member_manifest, canonical_bytes(manifest) + b"\n") + manifest_digest, manifest_size = sha256_file(args.member_manifest) + report = { + "schemaVersion": "phase4-native-build-v1", + "binary": {"name": name, "size": size, "sha256": digest}, + "memberManifest": {"name": args.member_manifest.name, "size": manifest_size, "sha256": manifest_digest}, + "sourceDateEpoch": os.environ.get("SOURCE_DATE_EPOCH"), + "macosDeploymentTarget": os.environ.get("MACOSX_DEPLOYMENT_TARGET"), + "cargoIncremental": os.environ.get("CARGO_INCREMENTAL"), + } + create_file_atomic(args.report, canonical_bytes(report) + b"\n") + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 + except (ForgeError, OSError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase4_payloads.py b/scripts/phase4_payloads.py new file mode 100755 index 0000000..b844362 --- /dev/null +++ b/scripts/phase4_payloads.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Bounded Phase-4 inspection/extraction for pinned upstream tarballs. + +The source archive is untrusted even after its reviewed SHA-256 is verified. This +tool admits only the three frozen Phase-4 layouts, extracts regular files into a +fresh directory without following links, normalizes the warehouse install modes, +and emits the exact member manifest consumed by package_deterministic.py. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import sys +import tarfile +from pathlib import Path + +from forge_io import ( + ForgeError, + canonical_bytes, + create_file_atomic, + expect, + safe_basename, + safe_member_name, + sha256_file, + validate_unique_names, +) + + +CONTRACTS = { + "node": {"maxCompressed": 128 * 2**20, "maxExpanded": 512 * 2**20, "maxMembers": 512}, + "toolkit": {"maxCompressed": 96 * 2**20, "maxExpanded": 256 * 2**20, "maxMembers": 1}, + "celestia-appd": {"maxCompressed": 256 * 2**20, "maxExpanded": 512 * 2**20, "maxMembers": 3}, + "celestia-node": {"maxCompressed": 128 * 2**20, "maxExpanded": 256 * 2**20, "maxMembers": 3}, +} + + +def classify(contract: str, names: set[str]) -> None: + if contract == "toolkit": + expect(names == {"midnight-node-toolkit"}, "toolkit source must contain one literal root executable") + elif contract == "node": + expect("midnight-node" in names, "node source is missing the literal root executable") + expect("res" in names, "node source is missing the res directory") + expect(all(name in {"midnight-node", "res"} or name.startswith("res/") for name in names), "node source contains a member outside midnight-node plus res/") + expect(any(name.startswith("res/") for name in names), "node res directory is empty") + else: + executable = "celestia-appd" if contract == "celestia-appd" else "celestia" + expect(names == {"LICENSE", "README.md", executable}, f"{contract} source must contain LICENSE, README.md, and {executable}") + + +def expected_mode(contract: str, name: str, is_dir: bool) -> str: + if is_dir: + return "0755" + if contract == "toolkit" or name == "midnight-node" or name in {"celestia-appd", "celestia"}: + return "0755" + return "0644" + + +def destination_name(contract: str, source_name: str, renamed_executable: str | None) -> str: + if contract == "node": + expect(renamed_executable is not None, "node extraction requires --renamed-executable") + if source_name == "midnight-node": + return safe_basename(renamed_executable, "renamed executable") + return source_name + expect(renamed_executable is None, "--renamed-executable applies only to node") + return source_name + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--contract", required=True, choices=sorted(CONTRACTS)) + parser.add_argument("--archive", required=True, type=Path) + parser.add_argument("--staging", required=True, type=Path) + parser.add_argument("--member-manifest", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--renamed-executable") + args = parser.parse_args() + try: + limits = CONTRACTS[args.contract] + archive_digest, compressed_size = sha256_file(args.archive, limits["maxCompressed"]) + expect(args.staging.is_dir() and not any(args.staging.iterdir()), "staging must be an empty existing directory") + expect(not args.member_manifest.exists() and not args.report.exists(), "refusing to replace evidence output") + + with tarfile.open(args.archive, mode="r:gz") as source: + expect(not source.pax_headers, "global PAX metadata is forbidden") + members = source.getmembers() + expect(0 < len(members) <= limits["maxMembers"], "source member-count ceiling exceeded") + normalized_names: list[str] = [] + by_name: dict[str, tarfile.TarInfo] = {} + expanded = 0 + for member in members: + expect(not member.pax_headers, f"PAX metadata is forbidden: {member.name}") + name = safe_member_name(member.name.rstrip("/")) + expect(member.isdir() or member.isreg(), f"unsafe/non-regular member type: {name}") + normalized_names.append(name) + by_name[name] = member + if member.isreg(): + expanded += member.size + expect(expanded <= limits["maxExpanded"], "source expanded-size ceiling exceeded") + validate_unique_names(normalized_names) + expect(len(by_name) == len(members), "duplicate normalized member path") + classify(args.contract, set(normalized_names)) + + output_rows: list[dict[str, object]] = [] + for source_name in sorted(normalized_names): + member = by_name[source_name] + target_name = destination_name(args.contract, source_name, args.renamed_executable) + target = args.staging / target_name + expect(target.resolve().is_relative_to(args.staging.resolve()), f"member escapes staging: {target_name}") + mode = expected_mode(args.contract, source_name, member.isdir()) + if member.isdir(): + target.mkdir(parents=True, exist_ok=False, mode=0o700) + os.chmod(target, int(mode, 8)) + output_rows.append({"path": target_name, "type": "directory", "mode": mode}) + continue + + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + stream = source.extractfile(member) + expect(stream is not None, f"cannot open source member: {source_name}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(target, flags, 0o600) + hasher = hashlib.sha256() + total = 0 + with stream, os.fdopen(descriptor, "wb") as output: + while True: + chunk = stream.read(min(1024 * 1024, member.size - total + 1)) + if not chunk: + break + total += len(chunk) + expect(total <= member.size, f"member exceeds declared size: {source_name}") + hasher.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + expect(total == member.size, f"member size mismatch: {source_name}") + os.chmod(target, int(mode, 8)) + output_rows.append({"path": target_name, "type": "file", "mode": mode, "size": total, "sha256": hasher.hexdigest()}) + + output_rows.sort(key=lambda row: str(row["path"])) + output_names = [str(row["path"]) for row in output_rows] + validate_unique_names(output_names) + actual_names = sorted(path.relative_to(args.staging).as_posix() for path in args.staging.rglob("*")) + expect(actual_names == output_names, "extracted tree differs from exact output manifest") + manifest = {"schemaVersion": "member-manifest-v1", "members": output_rows} + create_file_atomic(args.member_manifest, canonical_bytes(manifest) + b"\n") + manifest_sha256, manifest_size = sha256_file(args.member_manifest) + report = { + "schemaVersion": "phase4-source-inspection-v1", + "contract": args.contract, + "sourceArchive": {"name": args.archive.name, "size": compressed_size, "sha256": archive_digest}, + "expandedSize": expanded, + "memberCount": len(output_rows), + "memberManifest": {"name": args.member_manifest.name, "size": manifest_size, "sha256": manifest_sha256}, + } + create_file_atomic(args.report, canonical_bytes(report) + b"\n") + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 + except (ForgeError, OSError, tarfile.TarError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase4_sbom.sh b/scripts/phase4_sbom.sh new file mode 100755 index 0000000..f3d53ed --- /dev/null +++ b/scripts/phase4_sbom.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "usage: phase4_sbom.sh " >&2 + exit 2 +fi + +binary=$1 +output_prefix=$2 +os_name=$3 +arch=$4 +case "${os_name}/${arch}" in + linux/amd64) + asset=syft_1.51.1_linux_amd64.tar.gz + size=29203595 + digest=8fcb33017a0dc1058298c923c436d19dfa68ae93968e0b423248542e3afb9fc3 + ;; + linux/arm64) + asset=syft_1.51.1_linux_arm64.tar.gz + size=26668605 + digest=a7fd2b784e6664acd44719270574f6cd8c6864fc2b1700bf9099bd1cccda7d7f + ;; + macos/arm64) + asset=syft_1.51.1_darwin_arm64.tar.gz + size=27907057 + digest=ac063af3b9874769deb7ea1e6d76841e68f9e3bb50cd654226fc977de65532c1 + ;; + *) + echo "unsupported native Syft tuple: ${os_name}/${arch}" >&2 + exit 2 + ;; +esac + +scratch="${RUNNER_TEMP:?}/phase4-syft-${os_name}-${arch}" +test ! -e "$scratch" +mkdir -p "$scratch/download" "$scratch/tool" +python3 scripts/fetch_verified.py \ + --url "https://github.com/anchore/syft/releases/download/v1.51.1/${asset}" \ + --output "$scratch/download/$asset" \ + --size "$size" \ + --sha256 "$digest" +members=$(tar -tzf "$scratch/download/$asset") +printf '%s\n' "$members" | grep -Fxq syft +if printf '%s\n' "$members" | grep -Eq '(^/|(^|/)\.\.(/|$)|\\)'; then + echo 'unsafe Syft archive member' >&2 + exit 2 +fi +tar -xzf "$scratch/download/$asset" --no-same-owner --no-same-permissions -C "$scratch/tool" syft +chmod 0755 "$scratch/tool/syft" +"$scratch/tool/syft" version +mkdir -p "$(dirname "$output_prefix")" +"$scratch/tool/syft" scan "file:$binary" \ + -o "spdx-json=${output_prefix}.spdx.json" \ + -o "cyclonedx-json=${output_prefix}.cyclonedx.json" +python3 -m json.tool "${output_prefix}.spdx.json" >/dev/null +python3 -m json.tool "${output_prefix}.cyclonedx.json" >/dev/null diff --git a/scripts/probe_phase4_toolkit.sh b/scripts/probe_phase4_toolkit.sh new file mode 100755 index 0000000..2532d0e --- /dev/null +++ b/scripts/probe_phase4_toolkit.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo 'usage: probe_phase4_toolkit.sh ' >&2 + exit 2 +fi + +binary=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") +evidence=$2 +mkdir -p "$evidence" +work=$(mktemp -d "${RUNNER_TEMP:-/tmp}/phase4-toolkit-probe.XXXXXX") +trap 'find "$work" -depth -delete' EXIT + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +export LC_ALL=C +export TZ=UTC +cd "$work" +"$binary" version >"$evidence/version.stdout" 2>"$evidence/version.stderr" +test ! -s "$evidence/version.stderr" +grep -Fq '2.0.0' "$evidence/version.stdout" +grep -Fq '7.0.3' "$evidence/version.stdout" +grep -Fq '0.31.0-6587676a9bb2' "$evidence/version.stdout" +"$binary" --help >"$evidence/help.stdout" 2>"$evidence/help.stderr" +test ! -s "$evidence/help.stderr" + +"$binary" show-address \ + --network undeployed \ + --seed 0000000000000000000000000000000000000000000000000000000000000001 \ + --unshielded >"$evidence/show-address.stdout" 2>"$evidence/show-address.stderr" +test ! -s "$evidence/show-address.stderr" +printf 'mn_addr_undeployed1h3ssm5ru2t6eqy4g3she78zlxn96e36ms6pq996aduvmateh9p9sk96u7s\n' >"$work/expected-address" +cmp "$work/expected-address" "$evidence/show-address.stdout" +test "$(wc -c <"$evidence/show-address.stdout" | tr -d ' ')" = 78 +test "$(sha256_file "$evidence/show-address.stdout")" = 6387f62fb1d77d0b2ece8e2931265485cdb585a4d85fd32515229c64404206c8 + +set +e +"$binary" show-address --network undeployed --unshielded >"$evidence/missing-seed.stdout" 2>"$evidence/missing-seed.stderr" +status=$? +set -e +test "$status" = 2 +test ! -s "$evidence/missing-seed.stdout" +test "$(sha256_file "$evidence/missing-seed.stderr")" = 9f064fa173e49cfb3a74b161334e35d0ef787e200699b788c6921419ff01ca28 +printf 'missingSeedExit=%s\n' "$status" >"$evidence/probe-summary.txt" +for probe_file in "$evidence"/*; do + printf '%s %s\n' "$(sha256_file "$probe_file")" "$(basename "$probe_file")" >>"$evidence/probe-summary.txt" +done diff --git a/scripts/validate_phase4_contract.py b/scripts/validate_phase4_contract.py new file mode 100644 index 0000000..4c1a695 --- /dev/null +++ b/scripts/validate_phase4_contract.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Cross-bind the declared Phase-4 macOS build contract and emitted evidence.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +from forge_io import ForgeError, expect, load_json, sha256_file + + +COMPONENT = Path("catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json") +PINS = Path("evidence/phase4/source-pins.json") +WORKFLOW = Path(".github/workflows/phase4-payloads.yml") +TRANSFORMATION_RECORD = Path("evidence/phase4/transformation-toolchain.json") +EVIDENCE_TOOLCHAIN = Path("evidence/phase4/evidence-closure-toolchain.json") +COMPONENTS = Path("catalog/components") +LINUX_REPACKAGE_COMPONENTS = ( + "midnight-node-2.0.0-rc.4-linux-arm64.json", + "midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json", + "midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json", +) + + +def validate_tool_record(record_path: Path, root: Path, schema: str) -> str: + expect(record_path.is_file() and not record_path.is_symlink(), f"tool record missing or unsafe: {record_path}") + record = load_json(record_path) + expect(record["schemaVersion"] == schema, f"tool record schema mismatch: {record_path}") + rows = record["scripts"] + expect(isinstance(rows, list) and rows, f"tool record scripts missing: {record_path}") + paths = [row["path"] for row in rows] + expect(paths == sorted(paths) and len(paths) == len(set(paths)), f"tool record paths must be unique/sorted: {record_path}") + for row in rows: + relative = Path(row["path"]) + expect(not relative.is_absolute() and ".." not in relative.parts, f"unsafe tool record path: {row['path']}") + script = root / relative + expect(script.is_file() and not script.is_symlink(), f"tool record script missing or unsafe: {row['path']}") + expect(sha256_file(script)[0] == row["sha256"], f"tool record script digest mismatch: {row['path']}") + return sha256_file(record_path)[0] + + +def validate( + root: Path, + component_path: Path, + pins_path: Path, + workflow_path: Path, + transformation_record: Path, + evidence_toolchain: Path, + components_dir: Path, + native_report: Path | None, + payload_evidence: Path | None, +) -> None: + component = load_json(component_path) + pins = load_json(pins_path) + workflow = workflow_path.read_text(encoding="utf-8") + + source = component["source"] + toolkit = pins["node"]["toolkitSource"] + epoch = toolkit["sourceDateEpoch"] + derivation = toolkit["sourceDateEpochDerivation"] + commit = source["commitSha"] + transformation_digest = validate_tool_record(transformation_record, root, "phase4-transformation-toolchain-v1") + transformation_locator = f"forge-phase4-transformation-toolchain-v1@sha256:{transformation_digest}" + for name in LINUX_REPACKAGE_COMPONENTS: + linux_source = load_json(components_dir / name)["source"] + expect(linux_source["toolchainDigest"] == transformation_digest, f"component transformation record digest mismatch: {name}") + expect(linux_source["toolchain"] == transformation_locator, f"component transformation record locator mismatch: {name}") + + evidence_digest = validate_tool_record(evidence_toolchain, root, "phase4-evidence-closure-toolchain-v1") + evidence_locator = f"forge-phase4-evidence-closure-toolchain-v1@sha256:{evidence_digest}" + evidence_binding = toolkit["evidenceClosureToolchain"] + expect(evidence_binding["path"] == EVIDENCE_TOOLCHAIN.as_posix(), "evidence toolchain path mismatch") + expect(evidence_binding["sha256"] == evidence_digest, "evidence toolchain pin digest mismatch") + expect(evidence_binding["locator"] == evidence_locator, "evidence toolchain pin locator mismatch") + expect(source["buildFlags"].count(evidence_locator) == 1, "component evidence toolchain locator mismatch") + expect(isinstance(epoch, str) and re.fullmatch(r"[1-9][0-9]{9}", epoch) is not None, "sourceDateEpoch must be ten decimal seconds") + expect(derivation == f"git-commit-committer-unix-seconds:{commit}", "sourceDateEpoch derivation/commit mismatch") + expect(pins["node"]["commitSha"] == commit, "component/source-pin commit mismatch") + expected_flag = f"SOURCE_DATE_EPOCH={epoch}" + flags = source["buildFlags"] + expect(flags.count(expected_flag) == 1, "component must declare exact SOURCE_DATE_EPOCH once") + expect(toolkit["buildCommand"] in flags, "component/source-pin build command mismatch") + expect(f"cargo-auditable-wrapper-path:{toolkit['cargoAuditableWrapperPath']}" in flags, "component/source-pin wrapper path mismatch") + expect(f"MACOSX_DEPLOYMENT_TARGET={toolkit['macosDeploymentTarget']}" in flags, "component/source-pin deployment target mismatch") + + expect(workflow.count(f"SOURCE_DATE_EPOCH: '{epoch}'") == 1, "workflow SOURCE_DATE_EPOCH literal mismatch") + required_workflow_checks = ( + 'test "$SOURCE_DATE_EPOCH" = "$(git -C upstream show -s --format=%ct HEAD)"', + "'.node.toolkitSource.sourceDateEpoch'", + "'.node.toolkitSource.sourceDateEpochDerivation'", + "'.source.buildFlags[]'", + "'.sourceDateEpoch == $epoch'", + 'index("SOURCE_DATE_EPOCH=" + $epoch) != null', + ) + for fragment in required_workflow_checks: + expect(fragment in workflow, f"workflow Phase-4 epoch cross-check missing: {fragment}") + + if native_report is not None: + report = load_json(native_report) + expect(report["sourceDateEpoch"] == epoch, "native report SOURCE_DATE_EPOCH mismatch") + if payload_evidence is not None: + evidence = load_json(payload_evidence) + expect(evidence["source"]["buildFlags"].count(expected_flag) == 1, "payload evidence SOURCE_DATE_EPOCH mismatch") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--component", type=Path) + parser.add_argument("--pins", type=Path) + parser.add_argument("--workflow", type=Path) + parser.add_argument("--transformation-record", type=Path) + parser.add_argument("--evidence-toolchain", type=Path) + parser.add_argument("--components-dir", type=Path) + parser.add_argument("--native-report", type=Path) + parser.add_argument("--payload-evidence", type=Path) + args = parser.parse_args() + try: + validate( + args.root, + args.component or args.root / COMPONENT, + args.pins or args.root / PINS, + args.workflow or args.root / WORKFLOW, + args.transformation_record or args.root / TRANSFORMATION_RECORD, + args.evidence_toolchain or args.root / EVIDENCE_TOOLCHAIN, + args.components_dir or args.root / COMPONENTS, + args.native_report, + args.payload_evidence, + ) + print("OK Phase-4 macOS declared/emitted build contract") + return 0 + except (ForgeError, OSError, KeyError, TypeError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_phase4.py b/tests/test_phase4.py new file mode 100644 index 0000000..017b373 --- /dev/null +++ b/tests/test_phase4.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import hashlib +import io +import json +import os +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" + + +def run_script(name: str, *arguments: str, expected: int = 0) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [sys.executable, str(SCRIPTS / name), *arguments], + cwd=ROOT, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != expected: + raise AssertionError( + f"{name} returned {result.returncode}, expected {expected}\n" + f"stdout={result.stdout}\nstderr={result.stderr}" + ) + return result + + +def add_file(archive: tarfile.TarFile, name: str, data: bytes, mode: int) -> None: + info = tarfile.TarInfo(name) + info.size = len(data) + info.mode = mode + info.mtime = 1 + archive.addfile(info, io.BytesIO(data)) + + +def add_directory(archive: tarfile.TarFile, name: str, mode: int = 0o755) -> None: + info = tarfile.TarInfo(name) + info.type = tarfile.DIRTYPE + info.mode = mode + info.mtime = 1 + archive.addfile(info) + + +def write_source(path: Path, contract: str) -> None: + with tarfile.open(path, "w:gz", format=tarfile.GNU_FORMAT) as archive: + if contract == "toolkit": + add_file(archive, "midnight-node-toolkit", b"toolkit-fixture\n", 0o700) + elif contract == "node": + add_file(archive, "midnight-node", b"node-fixture\n", 0o700) + add_directory(archive, "res") + add_directory(archive, "res/chainspecs") + add_file(archive, "res/chainspecs/dev.json", b"{}\n", 0o600) + elif contract == "celestia-appd": + add_file(archive, "LICENSE", b"Apache-2.0 fixture\n", 0o600) + add_file(archive, "README.md", b"fixture\n", 0o600) + add_file(archive, "celestia-appd", b"appd-fixture\n", 0o700) + elif contract == "celestia-node": + add_file(archive, "LICENSE", b"Apache-2.0 fixture\n", 0o600) + add_file(archive, "README.md", b"fixture\n", 0o600) + add_file(archive, "celestia", b"node-fixture\n", 0o700) + else: + raise AssertionError(contract) + + +def extract_fixture(root: Path, contract: str, renamed: str | None = None) -> tuple[Path, Path, Path]: + archive = root / f"{contract}.tar.gz" + staging = root / f"{contract}-staging" + manifest = root / f"{contract}-members.json" + report = root / f"{contract}-report.json" + staging.mkdir() + write_source(archive, contract) + arguments = [ + "--contract", contract, + "--archive", str(archive), + "--staging", str(staging), + "--member-manifest", str(manifest), + "--report", str(report), + ] + if renamed is not None: + arguments.extend(("--renamed-executable", renamed)) + run_script("phase4_payloads.py", *arguments) + return staging, manifest, report + + +def file_identity(path: Path) -> dict[str, object]: + data = path.read_bytes() + return {"name": path.name, "size": len(data), "sha256": hashlib.sha256(data).hexdigest()} + + +def make_macos_build_fixture(root: Path) -> None: + payload = root / "payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip" + evidence = root / "evidence" + sbom = root / "sbom" + payload.parent.mkdir(parents=True) + evidence.mkdir() + sbom.mkdir() + payload.write_bytes(b"deterministic zip fixture\n") + contents = { + evidence / "build-and-system.log": b"build fixture\n", + evidence / "macos-signature.json": b'{"codeSignatureKind":"linker-adhoc"}\n', + evidence / "member-manifest.json": b'{"members":[]}\n', + evidence / "native-build-report.json": b'{"sourceDateEpoch":"1783616457"}\n', + evidence / "probe.log": b"probe fixture\n", + sbom / "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.cyclonedx.json": b'{"bomFormat":"CycloneDX"}\n', + sbom / "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.spdx.json": b'{"spdxVersion":"SPDX-2.3"}\n', + } + for path, data in contents.items(): + path.write_bytes(data) + record = { + "payload": file_identity(payload), + "evidence": sorted((file_identity(path) for path in contents), key=lambda row: str(row["name"])), + "signing": {"codeSignatureKind": "linker-adhoc"}, + "source": {"buildFlags": ["SOURCE_DATE_EPOCH=1783616457"]}, + } + payload_evidence = evidence / "payload-evidence.json" + payload_evidence.write_text(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + checksum_paths = [payload, payload_evidence, *contents] + rows = [f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n" for path in sorted(checksum_paths, key=lambda item: item.name)] + (evidence / "SHA256SUMS").write_text("".join(rows)) + + +class Phase4PayloadTest(unittest.TestCase): + def test_toolkit_extracts_one_literal_binary_and_repacks_reproducibly(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + staging, manifest, _ = extract_fixture(root, "toolkit") + binary = staging / "midnight-node-toolkit" + self.assertEqual(binary.read_bytes(), b"toolkit-fixture\n") + self.assertEqual(stat.S_IMODE(binary.stat().st_mode), 0o755) + self.assertEqual(json.loads(manifest.read_text())["members"], [{ + "mode": "0755", + "path": "midnight-node-toolkit", + "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "size": len(binary.read_bytes()), + "type": "file", + }]) + first = root / "first.zip" + second = root / "second.zip" + for output in (first, second): + run_script("package_deterministic.py", "--input-dir", str(staging), "--members", str(manifest), "--output", str(output)) + self.assertEqual(first.read_bytes(), second.read_bytes()) + with zipfile.ZipFile(first) as archive: + self.assertEqual(archive.namelist(), ["midnight-node-toolkit"]) + self.assertEqual((archive.getinfo("midnight-node-toolkit").external_attr >> 16) & 0o777, 0o755) + + def test_node_preserves_exact_res_tree_and_versions_root_executable(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + renamed = "midnight-node-linux-arm64-2.0.0-rc.4" + staging, manifest, report = extract_fixture(root, "node", renamed) + self.assertEqual( + sorted(path.relative_to(staging).as_posix() for path in staging.rglob("*")), + [renamed, "res", "res/chainspecs", "res/chainspecs/dev.json"], + ) + self.assertEqual(stat.S_IMODE((staging / renamed).stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((staging / "res/chainspecs/dev.json").stat().st_mode), 0o644) + self.assertEqual(json.loads(report.read_text())["memberCount"], 4) + output = root / "node.zip" + run_script("package_deterministic.py", "--input-dir", str(staging), "--members", str(manifest), "--output", str(output)) + with zipfile.ZipFile(output) as archive: + self.assertEqual( + archive.namelist(), + [renamed, "res/", "res/chainspecs/", "res/chainspecs/dev.json"], + ) + + def test_celestia_contracts_admit_only_three_literal_root_members(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for contract, executable in (("celestia-appd", "celestia-appd"), ("celestia-node", "celestia")): + staging, manifest, report = extract_fixture(root, contract) + self.assertEqual(sorted(path.name for path in staging.iterdir()), ["LICENSE", "README.md", executable]) + self.assertEqual(json.loads(report.read_text())["memberCount"], 3) + modes = {row["path"]: row["mode"] for row in json.loads(manifest.read_text())["members"]} + self.assertEqual(modes, {"LICENSE": "0644", "README.md": "0644", executable: "0755"}) + + def test_extractor_rejects_missing_res_links_and_unexpected_members(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases: list[tuple[str, str]] = [("missing-res", "node"), ("symlink", "toolkit"), ("extra", "celestia-node")] + for case, contract in cases: + archive_path = root / f"{case}.tar.gz" + with tarfile.open(archive_path, "w:gz", format=tarfile.GNU_FORMAT) as archive: + if case == "missing-res": + add_file(archive, "midnight-node", b"node", 0o755) + elif case == "symlink": + link = tarfile.TarInfo("midnight-node-toolkit") + link.type = tarfile.SYMTYPE + link.linkname = "/bin/true" + archive.addfile(link) + else: + add_file(archive, "LICENSE", b"license", 0o644) + add_file(archive, "README.md", b"readme", 0o644) + add_file(archive, "celestia", b"binary", 0o755) + add_file(archive, "unexpected", b"no", 0o644) + staging = root / f"{case}-staging" + staging.mkdir() + arguments = [ + "--contract", contract, + "--archive", str(archive_path), + "--staging", str(staging), + "--member-manifest", str(root / f"{case}-manifest.json"), + "--report", str(root / f"{case}-report.json"), + ] + if contract == "node": + arguments.extend(("--renamed-executable", "midnight-node-linux-arm64-2.0.0-rc.4")) + result = run_script("phase4_payloads.py", *arguments, expected=2) + self.assertIn("ERROR:", result.stderr) + + def test_phase4_exact_tuple_resolution_and_family_coverage(self) -> None: + manifests = [json.loads(path.read_text()) for path in sorted((ROOT / "catalog/components").glob("*.json"))] + phase4 = [row for row in manifests if row["componentId"] in { + "midnight-node-2.0.0-rc.4-linux-arm64", + "midnight-node-toolkit-2.0.0-rc.4-linux-amd64", + "midnight-node-toolkit-2.0.0-rc.4-linux-arm64", + "midnight-node-toolkit-2.0.0-rc.4-macos-arm64", + "celestia-appd-6.4.10-linux-arm64", + "celestia-node-0.28.4-linux-arm64", + }] + self.assertEqual(len(phase4), 6) + candidates: dict[tuple[str, str, str, str], str] = {} + for component in phase4: + self.assertEqual(component["distributionTier"], "development-only") + self.assertEqual(component["releaseMutability"], "mutable-warehouse") + target = component["targets"][0] + name = component["naming"]["outerTemplate"].format(version=component["version"], os=target["os"], arch=target["arch"]) + key = (component["family"], component["version"], target["os"], target["arch"]) + self.assertNotIn(key, candidates) + candidates[key] = name + self.assertEqual(set(candidates.values()), { + "midnight-node-linux-arm64-2.0.0-rc.4.zip", + "midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip", + "midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip", + "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip", + "celestia-appd-linux-arm64-v6.4.10.tar.gz", + "celestia-node-linux-arm64-v0.28.4.tar.gz", + }) + self.assertNotIn(("midnight-node-toolkit", "2.0.0-rc.4", "darwin", "arm64"), candidates) + self.assertNotIn(("midnight-node-toolkit", "2.0.0-rc.4", "linux", "aarch64"), candidates) + + existing = {row["name"] for row in json.loads((ROOT / "evidence/phase0/warehouse-release-0.3.120.json").read_text())["assets"]} + combined = existing | set(candidates.values()) + families = { + ("midnight-node", "2.0.0-rc.4"): "midnight-node-{os}-{arch}-2.0.0-rc.4.zip", + ("midnight-node-toolkit", "2.0.0-rc.4"): "midnight-node-toolkit-{os}-{arch}-2.0.0-rc.4.zip", + ("celestia-appd", "6.4.10"): "celestia-appd-{os}-{arch}-v6.4.10.tar.gz", + ("celestia-node", "0.28.4"): "celestia-node-{os}-{arch}-v0.28.4.tar.gz", + } + for family, template in families.items(): + with self.subTest(family=family): + self.assertIn(template.format(os="linux", arch="amd64"), combined) + self.assertIn(template.format(os="macos", arch="arm64"), combined) + self.assertIn(template.format(os="linux", arch="arm64"), combined) + + def test_native_build_attempt_selector_is_not_exported_to_cargo(self) -> None: + workflow = (ROOT / ".github/workflows/phase4-payloads.yml").read_text() + self.assertNotIn("BUILD_ID:", workflow) + self.assertIn("build_id='${{ matrix.build_id }}'", workflow) + self.assertIn('case "$build_id" in 1|2)', workflow) + self.assertIn('work="$RUNNER_TEMP/phase4-macos-build"', workflow) + self.assertNotIn('phase4-macos-build-$build_id', workflow) + command = "cargo auditable rustc --locked --release --no-default-features -p midnight-node-toolkit --bin midnight-node-toolkit -- -C link-arg=-Wl,-no_uuid" + self.assertEqual(workflow.count(command), 2) + self.assertIn("grep -Fq 'cmd LC_UUID'", workflow) + + def test_macos_build_contract_cross_binding_and_mutations(self) -> None: + component_path = ROOT / "catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json" + pins_path = ROOT / "evidence/phase4/source-pins.json" + workflow_path = ROOT / ".github/workflows/phase4-payloads.yml" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + component = root / "component.json" + pins = root / "pins.json" + workflow = root / "workflow.yml" + transformation_record = root / "transformation-toolchain.json" + evidence_toolchain = root / "evidence-closure-toolchain.json" + components_dir = root / "components" + report = root / "native-build-report.json" + payload_evidence = root / "payload-evidence.json" + components_dir.mkdir() + component.write_bytes(component_path.read_bytes()) + pins.write_bytes(pins_path.read_bytes()) + workflow.write_bytes(workflow_path.read_bytes()) + transformation_record.write_bytes((ROOT / "evidence/phase4/transformation-toolchain.json").read_bytes()) + evidence_toolchain.write_bytes((ROOT / "evidence/phase4/evidence-closure-toolchain.json").read_bytes()) + linux_names = ( + "midnight-node-2.0.0-rc.4-linux-arm64.json", + "midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json", + "midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json", + ) + for name in linux_names: + (components_dir / name).write_bytes((ROOT / "catalog/components" / name).read_bytes()) + report.write_text('{"sourceDateEpoch":"1783616457"}\n') + flags = json.loads(component.read_text())["source"]["buildFlags"] + payload_evidence.write_text(json.dumps({"source": {"buildFlags": flags}}) + "\n") + arguments = ( + "--component", str(component), "--pins", str(pins), "--workflow", str(workflow), + "--transformation-record", str(transformation_record), + "--evidence-toolchain", str(evidence_toolchain), "--components-dir", str(components_dir), + "--native-report", str(report), "--payload-evidence", str(payload_evidence), + ) + run_script("validate_phase4_contract.py", *arguments) + + mutations: list[tuple[str, Path, bytes]] = [] + mutated_pins = json.loads(pins.read_text()) + mutated_pins["node"]["toolkitSource"]["sourceDateEpoch"] = "1783616458" + mutations.append(("pins", pins, (json.dumps(mutated_pins) + "\n").encode())) + mutated_component = json.loads(component.read_text()) + mutated_component["source"]["buildFlags"].remove("SOURCE_DATE_EPOCH=1783616457") + mutations.append(("component", component, (json.dumps(mutated_component) + "\n").encode())) + mutations.append(("workflow", workflow, workflow.read_bytes().replace(b"SOURCE_DATE_EPOCH: '1783616457'", b"SOURCE_DATE_EPOCH: '1783616458'", 1))) + mutations.append(("native-report", report, b'{"sourceDateEpoch":"1783616458"}\n')) + mutations.append(("payload-evidence", payload_evidence, b'{"source":{"buildFlags":[]}}\n')) + mutations.append(("transformation-record", transformation_record, transformation_record.read_bytes() + b" ")) + mutations.append(("evidence-toolchain-record", evidence_toolchain, evidence_toolchain.read_bytes() + b" ")) + linux_component = components_dir / linux_names[0] + mutated_linux = json.loads(linux_component.read_text()) + fake_digest = "f" * 64 + mutated_linux["source"]["toolchainDigest"] = fake_digest + mutated_linux["source"]["toolchain"] = f"forge-phase4-transformation-toolchain-v1@sha256:{fake_digest}" + mutations.append(("self-consistent-stale-component-record", linux_component, (json.dumps(mutated_linux) + "\n").encode())) + originals = {path: path.read_bytes() for _, path, _ in mutations} + for name, path, data in mutations: + with self.subTest(mutation=name): + path.write_bytes(data) + result = run_script("validate_phase4_contract.py", *arguments, expected=2) + self.assertIn("ERROR:", result.stderr) + path.write_bytes(originals[path]) + + def test_complete_macos_two_build_closure_and_negatives(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + build1 = root / "build1" + build2 = root / "build2" + make_macos_build_fixture(build1) + make_macos_build_fixture(build2) + comparison = root / "comparison.log" + comparison.write_text("independentBuildDigestMatch=true\ncleanRuntimeProbe=PASS\n") + output = root / "consolidated" + run_script( + "consolidate_phase4_macos.py", "assemble", + "--build1", str(build1), "--build2", str(build2), + "--comparison-log", str(comparison), "--output", str(output), + ) + run_script("consolidate_phase4_macos.py", "verify", "--root", str(output)) + root_rows = (output / "SHA256SUMS").read_text().splitlines() + expected_files = [path for path in output.rglob("*") if path.is_file() and path != output / "SHA256SUMS"] + self.assertEqual(len(root_rows), len(expected_files)) + self.assertTrue(any("independent-builds/build2/evidence/build-and-system.log" in row for row in root_rows)) + self.assertTrue(any("independent-builds/build2/sbom/" in row for row in root_rows)) + + for name, mutation in ( + ("missing", lambda tree: (tree / "independent-builds/build2/evidence/probe.log").unlink()), + ("extra", lambda tree: (tree / "independent-builds/build2/evidence/extra.log").write_text("extra\n")), + ("mutated", lambda tree: (tree / "independent-builds/build2/evidence/probe.log").write_text("changed\n")), + ): + with self.subTest(mutation=name): + mutated = root / f"negative-{name}" + shutil.copytree(output, mutated) + mutation(mutated) + self.assertIn("ERROR:", run_script("consolidate_phase4_macos.py", "verify", "--root", str(mutated), expected=2).stderr) + + dangling = root / "negative-dangling" + shutil.copytree(output, dangling) + build2_root = dangling / "independent-builds/build2" + record_path = build2_root / "evidence/payload-evidence.json" + record = json.loads(record_path.read_text()) + record["evidence"].append({"name": "missing.json", "size": 1, "sha256": "0" * 64}) + record_path.write_text(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + inner_sums = build2_root / "evidence/SHA256SUMS" + inner_sums.write_text("".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n" + for path in sorted((path for path in build2_root.rglob("*") if path.is_file() and path != inner_sums), key=lambda item: item.name) + )) + root_sum = dangling / "SHA256SUMS" + root_sum.write_text("".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.relative_to(dangling).as_posix()}\n" + for path in sorted((path for path in dangling.rglob("*") if path.is_file() and path != root_sum), key=lambda item: item.relative_to(dangling).as_posix()) + )) + result = run_script("consolidate_phase4_macos.py", "verify", "--root", str(dangling), expected=2) + self.assertIn("dangling payload evidence reference", result.stderr) + + def test_source_pins_and_transformation_script_digests_are_closed(self) -> None: + pins = json.loads((ROOT / "evidence/phase4/source-pins.json").read_text()) + self.assertEqual(pins["node"]["commitSha"], "651e043b61ed445bf7a5066c60c87ea7bd606073") + self.assertEqual(pins["node"]["treeSha"], "5c34f67538f20811d876f6463cf9aca5a3bc4fc9") + self.assertEqual(pins["node"]["rustToolchain"]["resolved"], "1.95.0") + self.assertEqual(pins["node"]["toolkitSource"]["defaultFeatures"], []) + self.assertEqual(pins["node"]["toolkitSource"]["forbiddenFeatures"], ["erase-proof"]) + self.assertEqual( + pins["node"]["toolkitSource"]["buildCommand"], + "cargo auditable rustc --locked --release --no-default-features -p midnight-node-toolkit --bin midnight-node-toolkit -- -C link-arg=-Wl,-no_uuid", + ) + self.assertEqual( + pins["node"]["toolkitSource"]["cargoAuditableWrapperPath"], + "$RUNNER_TEMP/phase4-macos-build/tool/cargo-auditable", + ) + self.assertEqual(pins["node"]["toolkitSource"]["sourceDateEpoch"], "1783616457") + self.assertEqual( + pins["node"]["toolkitSource"]["sourceDateEpochDerivation"], + "git-commit-committer-unix-seconds:651e043b61ed445bf7a5066c60c87ea7bd606073", + ) + self.assertEqual(pins["celestiaApp"]["license"]["spdx"], "Apache-2.0") + self.assertEqual(pins["celestiaNode"]["license"]["spdx"], "Apache-2.0") + transformation_path = ROOT / "evidence/phase4/transformation-toolchain.json" + transformation_digest = hashlib.sha256(transformation_path.read_bytes()).hexdigest() + self.assertEqual(transformation_digest, "141140312f43ea071a0f6cc50bf6374f6c0e1437651089eecd65a6b9369b936e") + for name in ( + "midnight-node-2.0.0-rc.4-linux-arm64.json", + "midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json", + "midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json", + ): + source = json.loads((ROOT / "catalog/components" / name).read_text())["source"] + self.assertEqual(source["toolchainDigest"], transformation_digest) + self.assertEqual(source["toolchain"], f"forge-phase4-transformation-toolchain-v1@sha256:{transformation_digest}") + evidence_path = ROOT / "evidence/phase4/evidence-closure-toolchain.json" + evidence_digest = hashlib.sha256(evidence_path.read_bytes()).hexdigest() + evidence_binding = pins["node"]["toolkitSource"]["evidenceClosureToolchain"] + self.assertEqual(evidence_binding["sha256"], evidence_digest) + self.assertEqual(evidence_binding["locator"], f"forge-phase4-evidence-closure-toolchain-v1@sha256:{evidence_digest}") + for toolchain_path in (transformation_path, evidence_path): + toolchain = json.loads(toolchain_path.read_text()) + for row in toolchain["scripts"]: + self.assertEqual(hashlib.sha256((ROOT / row["path"]).read_bytes()).hexdigest(), row["sha256"]) + + +if __name__ == "__main__": + unittest.main()