diff --git a/.env.sample.hoodi b/.env.sample.hoodi index 56400c3b..32593e49 100644 --- a/.env.sample.hoodi +++ b/.env.sample.hoodi @@ -101,7 +101,7 @@ LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://checkpoint-sync.hoodi.ethpandaops.io/ # Nimbus validator client docker container image version. # See available tags https://hub.docker.com/r/statusim/nimbus-validator-client/tags -#VC_NIMBUS_VERSION= +#VC_NIMBUS_VERSION=multiarch-v26.5.0 # Prysm validator client docker container image version. # See available tags https://hub.docker.com/r/offchainlabs/prysm-validator/tags diff --git a/.env.sample.mainnet b/.env.sample.mainnet index 2758a65b..7f3e829f 100644 --- a/.env.sample.mainnet +++ b/.env.sample.mainnet @@ -101,7 +101,7 @@ LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://mainnet.checkpoint.sigp.io/ # Nimbus validator client docker container image version. # See available tags https://hub.docker.com/r/statusim/nimbus-validator-client/tags -#VC_NIMBUS_VERSION= +#VC_NIMBUS_VERSION=multiarch-v26.5.0 # Prysm validator client docker container image version. # See available tags https://hub.docker.com/r/offchainlabs/prysm-validator/tags diff --git a/compose-cl.yml b/compose-cl.yml index c2e79974..7ff0d3b3 100644 --- a/compose-cl.yml +++ b/compose-cl.yml @@ -191,7 +191,7 @@ services: cl-nimbus: profiles: [ cl-nimbus ] - image: statusim/nimbus-eth2:${CL_NIMBUS_VERSION:-multiarch-v26.3.1} + image: statusim/nimbus-eth2:${CL_NIMBUS_VERSION:-multiarch-v26.5.0} restart: unless-stopped labels: - "alloy-monitored=${CL_NIMBUS_ALLOY_MONITORED:-true}" diff --git a/compose-vc.yml b/compose-vc.yml index 1179c547..369f0177 100644 --- a/compose-vc.yml +++ b/compose-vc.yml @@ -34,24 +34,51 @@ services: # | | | | | | | | | | |_) | |_| \__ \ # |_| |_|_|_| |_| |_|_.__/ \__,_|___/ + nimbus-beacon-bin: + profiles: [vc-nimbus] + image: statusim/nimbus-eth2:${VC_NIMBUS_VERSION:-multiarch-v26.5.0} + volumes: + - nimbus_beacon_node_bin:/out + entrypoint: + [ + "sh", + "-c", + "cp -f /home/user/nimbus_beacon_node /out/nimbus_beacon_node && chmod +x /out/nimbus_beacon_node", + ] + restart: "no" + + # Maintenance-only (ASDB export/import). Not started by compose up; use: + # docker compose --profile nimbus-maint run --rm nimbus-beacon ... + nimbus-beacon: + profiles: [nimbus-maint] + image: statusim/nimbus-eth2:${VC_NIMBUS_VERSION:-multiarch-v26.5.0} + networks: [dvnode] + volumes: + - ./data/vc-nimbus:/home/user/data + restart: "no" + vc-nimbus: profiles: [vc-nimbus] - image: "nimbus-validator-client:${VC_NIMBUS_VERSION:-multiarch-v26.2.1}" - build: - context: nimbus - args: - VERSION: ${VC_NIMBUS_VERSION:-multiarch-v26.2.1} - depends_on: [charon] + image: statusim/nimbus-validator-client:${VC_NIMBUS_VERSION:-multiarch-v26.5.0} + depends_on: + charon: + condition: service_started + nimbus-beacon-bin: + condition: service_completed_successfully networks: [dvnode] + entrypoint: ["/bin/bash", "/home/user/run.sh"] environment: BEACON_NODE_ADDRESS: http://charon:3600 BUILDER_API_ENABLED: ${BUILDER_API_ENABLED:-true} + NIMBUS_BEACON_NODE: /home/user/bin/nimbus_beacon_node labels: - "alloy-monitored=${VC_NIMBUS_ALLOY_MONITORED:-true}" volumes: - - ./nimbus/run.sh:/home/user/data/run.sh - - .charon/validator_keys:/home/validator_keys + - ./nimbus/run.sh:/home/user/run.sh:ro + - ./nimbus/import-keys.sh:/import-keys.sh:ro + - .charon/validator_keys:/home/validator_keys:ro - ./data/vc-nimbus:/home/user/data + - nimbus_beacon_node_bin:/home/user/bin:ro restart: unless-stopped # _ __ _ __ _ _ ___ _ __ ___ @@ -106,3 +133,6 @@ services: - .charon/validator_keys:/opt/charon/validator_keys - ./data/vc-teku:/home/data restart: unless-stopped + +volumes: + nimbus_beacon_node_bin: diff --git a/nimbus/Dockerfile b/nimbus/Dockerfile deleted file mode 100755 index ad27351a..00000000 --- a/nimbus/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG VERSION - -FROM statusim/nimbus-eth2:${VERSION} AS nimbusbn - -FROM statusim/nimbus-validator-client:${VERSION} - -COPY --from=nimbusbn /home/user/nimbus_beacon_node /home/user/nimbus_beacon_node - -ENTRYPOINT ["/home/user/data/run.sh"] diff --git a/nimbus/import-keys.sh b/nimbus/import-keys.sh new file mode 100644 index 00000000..24f6b139 --- /dev/null +++ b/nimbus/import-keys.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Import validator keystores into the Nimbus VC data directory (idempotent). +# Requires nimbus_beacon_node (from nimbus-eth2) at NIMBUS_BEACON_NODE. + +set -euo pipefail + +DATA_DIR="${DATA_DIR:-/home/user/data}" +VALIDATOR_KEYS_DIR="${VALIDATOR_KEYS_DIR:-/home/validator_keys}" +BEACON_NODE_BIN="${NIMBUS_BEACON_NODE:-/home/user/nimbus_beacon_node}" + +if find "${DATA_DIR}/validators" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | grep -q .; then + echo "Validators already present in ${DATA_DIR}/validators; skipping key import." + exit 0 +fi + +if [ ! -x "${BEACON_NODE_BIN}" ]; then + echo "Error: nimbus_beacon_node not found at ${BEACON_NODE_BIN}" >&2 + exit 1 +fi + +mkdir -p "${DATA_DIR}/validators" + +tmpkeys="${VALIDATOR_KEYS_DIR}/tmpkeys" +mkdir -p "${tmpkeys}" + +imported=0 +for f in "${VALIDATOR_KEYS_DIR}"/keystore-*.json; do + [ -e "$f" ] || continue + echo "Importing key ${f}" + + password=$(<"${f//json/txt}") + cp "${f}" "${tmpkeys}" + + echo "$password" | + "${BEACON_NODE_BIN}" deposits import \ + --data-dir="${DATA_DIR}" \ + "${tmpkeys}" + + rm "${tmpkeys}/$(basename "${f}")" + imported=$((imported + 1)) +done + +rm -rf "${tmpkeys}" + +if [ "$imported" -eq 0 ]; then + echo "Error: no keystore-*.json files found in ${VALIDATOR_KEYS_DIR}" >&2 + exit 1 +fi + +echo "Imported ${imported} key(s) into ${DATA_DIR}" diff --git a/nimbus/run.sh b/nimbus/run.sh index 464f535b..a915f30d 100755 --- a/nimbus/run.sh +++ b/nimbus/run.sh @@ -1,46 +1,26 @@ #!/usr/bin/env bash +# Start the Nimbus validator client. Preserves ./data/vc-nimbus (no wipe on restart). +# Imports keys automatically when ./data/vc-nimbus/validators is empty. -# Cleanup nimbus directories if they already exist. -rm -rf /home/user/data +set -euo pipefail -# Refer: https://nimbus.guide/keys.html -# Running a nimbus VC involves two steps which need to run in order: -# 1. Importing the validator keys -# 2. And then actually running the VC -tmpkeys="/home/validator_keys/tmpkeys" -mkdir -p ${tmpkeys} +DATA_DIR="/home/user/data" +export DATA_DIR +export VALIDATOR_KEYS_DIR="${VALIDATOR_KEYS_DIR:-/home/validator_keys}" +export NIMBUS_BEACON_NODE="${NIMBUS_BEACON_NODE:-/home/user/bin/nimbus_beacon_node}" -for f in /home/validator_keys/keystore-*.json; do - echo "Importing key ${f}" +bash /import-keys.sh - # Read password from keystore-*.txt into $password variable. - password=$(<"${f//json/txt}") +if ! find "${DATA_DIR}/validators" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | grep -q .; then + echo "Error: key import completed but no validators found in ${DATA_DIR}/validators." >&2 + exit 1 +fi - # Copy keystore file to tmpkeys/ directory. - cp "${f}" "${tmpkeys}" - - # Import keystore with the password. - echo "$password" | - /home/user/nimbus_beacon_node deposits import \ - --data-dir=/home/user/data \ - /home/validator_keys/tmpkeys - - # Delete tmpkeys/keystore-*.json file that was copied before. - filename="$(basename ${f})" - rm "${tmpkeys}/${filename}" -done - -# Delete the tmpkeys/ directory since it's no longer needed. -rm -r ${tmpkeys} - -echo "Imported all keys" - -# Now run nimbus VC exec /home/user/nimbus_validator_client \ - --data-dir=/home/user/data \ + --data-dir="${DATA_DIR}" \ --beacon-node="${BEACON_NODE_ADDRESS}" \ --doppelganger-detection=false \ --metrics \ --metrics-address=0.0.0.0 \ - --payload-builder=${BUILDER_API_ENABLED} \ + --payload-builder="${BUILDER_API_ENABLED}" \ --distributed diff --git a/prometheus/prometheus.yml.example b/prometheus/prometheus.yml.example index bdb3b85b..6d087529 100644 --- a/prometheus/prometheus.yml.example +++ b/prometheus/prometheus.yml.example @@ -45,7 +45,11 @@ scrape_configs: - job_name: "validator-client" static_configs: - - targets: ["lodestar:5064","vc-lodestar:5064"] + - targets: + - "lodestar:5064" + - "vc-lodestar:5064" + - "vc-nimbus:8008" + - "vc-teku:8008" # Debug - job_name: "node-exporter" static_configs: diff --git a/scripts/edit/vc/nimbus/export_asdb.sh b/scripts/edit/vc/nimbus/export_asdb.sh index 12d31b47..4bffafc7 100755 --- a/scripts/edit/vc/nimbus/export_asdb.sh +++ b/scripts/edit/vc/nimbus/export_asdb.sh @@ -9,7 +9,7 @@ # Usage: export_asdb.sh [--data-dir ] [--output-file ] # # Options: -# --data-dir Path to Nimbus data directory (default: ./data/nimbus) +# --data-dir Path to Nimbus data directory (default: ./data/vc-nimbus) # --output-file Path for exported slashing protection JSON (default: ./asdb-export/slashing-protection.json) # # Requirements: @@ -20,7 +20,7 @@ set -euo pipefail # Default values -DATA_DIR="./data/nimbus" +DATA_DIR="./data/vc-nimbus" OUTPUT_FILE="./asdb-export/slashing-protection.json" # Parse arguments @@ -94,17 +94,18 @@ OUTPUT_DIR=$(dirname "$OUTPUT_FILE") # Create output directory if it doesn't exist mkdir -p "$OUTPUT_DIR" -echo "Exporting slashing protection data using vc-nimbus container..." +# nimbus-beacon uses profile nimbus-maint (not started by compose up). +export COMPOSE_PROFILES="${COMPOSE_PROFILES}${COMPOSE_PROFILES:+,}nimbus-maint" -# Export slashing protection data using a temporary container based on vc-nimbus service. -# Note: slashingdb commands are in nimbus_beacon_node, not nimbus_validator_client. -# Nimbus requires --data-dir BEFORE the subcommand. -# We use docker compose run to create a temporary container with the same volumes. +echo "Exporting slashing protection data using nimbus-beacon service..." + +# slashingdb lives in nimbus_beacon_node (statusim/nimbus-eth2), not the VC image. if ! docker compose run --rm -T \ + --profile nimbus-maint \ -v "$OUTPUT_DIR":/tmp/asdb-export \ --entrypoint /home/user/nimbus_beacon_node \ - vc-nimbus --data-dir=/home/user/data slashingdb export /tmp/asdb-export/slashing-protection.json; then - echo "Error: Failed to export slashing protection from vc-nimbus" >&2 + nimbus-beacon --data-dir=/home/user/data slashingdb export /tmp/asdb-export/slashing-protection.json; then + echo "Error: Failed to export slashing protection from Nimbus data directory" >&2 exit 1 fi diff --git a/scripts/edit/vc/nimbus/import_asdb.sh b/scripts/edit/vc/nimbus/import_asdb.sh index f538491f..53da21f0 100755 --- a/scripts/edit/vc/nimbus/import_asdb.sh +++ b/scripts/edit/vc/nimbus/import_asdb.sh @@ -10,7 +10,7 @@ # # Options: # --input-file Path to updated slashing protection JSON (default: ./asdb-export/slashing-protection.json) -# --data-dir Path to Nimbus data directory (default: ./data/nimbus) +# --data-dir Path to Nimbus data directory (default: ./data/vc-nimbus) # # Requirements: # - .env file must exist with NETWORK variable set @@ -22,7 +22,7 @@ set -euo pipefail # Default values INPUT_FILE="./asdb-export/slashing-protection.json" -DATA_DIR="./data/nimbus" +DATA_DIR="./data/vc-nimbus" # Parse arguments while [[ $# -gt 0 ]]; do @@ -105,17 +105,18 @@ if docker compose ps --format '{{.Status}}' vc-nimbus 2>/dev/null | grep -qi run exit 1 fi -echo "Importing slashing protection data into vc-nimbus container..." +# nimbus-beacon uses profile nimbus-maint (not started by compose up). +export COMPOSE_PROFILES="${COMPOSE_PROFILES}${COMPOSE_PROFILES:+,}nimbus-maint" -# Import slashing protection data using a temporary container based on the vc-nimbus service. -# The input file is bind-mounted into the container at /tmp/import.json (read-only). -# Note: slashingdb commands are in nimbus_beacon_node, not nimbus_validator_client. -# Nimbus requires --data-dir BEFORE the subcommand. +echo "Importing slashing protection data into Nimbus data directory..." + +# slashingdb lives in nimbus_beacon_node (statusim/nimbus-eth2), not the VC image. if ! docker compose run --rm -T \ + --profile nimbus-maint \ --entrypoint sh \ -v "$INPUT_FILE":/tmp/import.json:ro \ - vc-nimbus -c "/home/user/nimbus_beacon_node --data-dir=/home/user/data slashingdb import /tmp/import.json"; then - echo "Error: Failed to import slashing protection into vc-nimbus container" >&2 + nimbus-beacon -c "/home/user/nimbus_beacon_node --data-dir=/home/user/data slashingdb import /tmp/import.json"; then + echo "Error: Failed to import slashing protection into Nimbus data directory" >&2 exit 1 fi diff --git a/scripts/edit/vc/test/docker-compose.test.yml b/scripts/edit/vc/test/docker-compose.test.yml index 8e694c80..be96beee 100644 --- a/scripts/edit/vc/test/docker-compose.test.yml +++ b/scripts/edit/vc/test/docker-compose.test.yml @@ -23,15 +23,27 @@ services: - ./scripts/edit/vc/test/fixtures/validator_keys:/home/charon/validator_keys - ./scripts/edit/vc/test/data/lodestar:/opt/data - vc-nimbus: + nimbus-beacon-bin: depends_on: [] + + nimbus-beacon: + profiles: [nimbus-maint] + volumes: + - ./nimbus/import-keys.sh:/import-keys.sh:ro + - ./scripts/edit/vc/test/fixtures/validator_keys:/home/validator_keys:ro + - ./scripts/edit/vc/test/data/nimbus:/home/user/data + + vc-nimbus: + depends_on: + nimbus-beacon-bin: + condition: service_completed_successfully entrypoint: ["sh", "-c", "tail -f /dev/null"] volumes: - # Mount run.sh from INSIDE the test data directory to avoid conflicts - # with the base compose's run.sh mount (volumes are merged, not replaced) - - ./scripts/edit/vc/test/data/nimbus/run.sh:/home/user/data/run.sh - - ./scripts/edit/vc/test/fixtures/validator_keys:/home/validator_keys + - ./nimbus/run.sh:/home/user/run.sh:ro + - ./nimbus/import-keys.sh:/import-keys.sh:ro + - ./scripts/edit/vc/test/fixtures/validator_keys:/home/validator_keys:ro - ./scripts/edit/vc/test/data/nimbus:/home/user/data + - nimbus_beacon_node_bin:/home/user/bin:ro vc-prysm: depends_on: [] diff --git a/scripts/edit/vc/test/test_nimbus_asdb.sh b/scripts/edit/vc/test/test_nimbus_asdb.sh index 298165d6..8b6a7c04 100755 --- a/scripts/edit/vc/test/test_nimbus_asdb.sh +++ b/scripts/edit/vc/test/test_nimbus_asdb.sh @@ -3,7 +3,7 @@ # Integration test for export/import ASDB scripts with Nimbus VC. # # This script: -# 1. Builds vc-nimbus image if needed +# 1. Pulls vc-nimbus / nimbus-beacon images if needed # 2. Starts vc-nimbus via docker-compose with test override (no charon dependency) # 3. Sets up keystores in the container # 4. Stops container and imports sample slashing protection data @@ -52,11 +52,6 @@ COMPOSE_FILE="$TEST_COMPOSE_FILES" docker compose --profile vc-nimbus down 2>/de rm -rf "$TEST_DATA_DIR" mkdir -p "$TEST_DATA_DIR" -# Copy run.sh into test data directory to satisfy the volume mount from base compose -# (compose merge keeps the original mount ./nimbus/run.sh:/home/user/data/run.sh, -# which conflicts with our test data mount unless we provide the file there) -cp "$REPO_ROOT/nimbus/run.sh" "$TEST_DATA_DIR/run.sh" - # Check prerequisites log_info "Checking prerequisites..." @@ -98,18 +93,21 @@ log_info "Using compose files: $COMPOSE_FILE" # Create test output directory mkdir -p "$TEST_OUTPUT_DIR" -# Step 0: Build vc-nimbus image if needed -log_info "Step 0: Building vc-nimbus image..." +# Step 0: Pull images +log_info "Step 0: Pulling Nimbus images..." -if ! docker compose --profile vc-nimbus build vc-nimbus; then - log_error "Failed to build vc-nimbus image" +if ! docker compose --profile vc-nimbus pull nimbus-beacon-bin vc-nimbus; then + log_error "Failed to pull Nimbus images" exit 1 fi -log_info "Image built successfully" +log_info "Images ready" -# Step 1: Start vc-nimbus via docker-compose -log_info "Step 1: Starting vc-nimbus via docker-compose..." +# Step 1: Stage beacon binary, import keys, then start vc-nimbus +log_info "Step 1: Staging beacon binary, importing keys, starting vc-nimbus..." +docker compose --profile vc-nimbus run --rm nimbus-beacon-bin +export COMPOSE_PROFILES="${COMPOSE_PROFILES},nimbus-maint" +docker compose --profile nimbus-maint run --rm --entrypoint /bin/bash nimbus-beacon /import-keys.sh docker compose --profile vc-nimbus up -d vc-nimbus sleep 2 @@ -123,36 +121,15 @@ fi log_info "Container started successfully" -# Step 2: Set up keystores using nimbus_beacon_node deposits import -log_info "Step 2: Setting up keystores..." +# Step 2: Verify keystores were imported +log_info "Step 2: Verifying keystores..." -# Create a temporary directory in the container for importing -docker compose exec -T vc-nimbus sh -c ' - mkdir -p /home/user/data/validators /tmp/keyimport - - for f in /home/validator_keys/keystore-*.json; do - echo "Importing key from $f" - - # Read password - password=$(cat "${f%.json}.txt") - - # Copy keystore to temp dir - cp "$f" /tmp/keyimport/ - - # Import using nimbus_beacon_node - echo "$password" | /home/user/nimbus_beacon_node deposits import \ - --data-dir=/home/user/data \ - /tmp/keyimport - - # Clean temp dir - rm /tmp/keyimport/* - done - - rm -rf /tmp/keyimport - echo "Done importing keystores" -' +if ! find "$TEST_DATA_DIR/validators" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | grep -q .; then + log_error "No validators found in $TEST_DATA_DIR/validators after import" + exit 1 +fi -log_info "Keystores set up successfully" +log_info "Keystores verified" # Step 3: Stop container and import sample slashing protection data log_info "Step 3: Importing sample slashing protection data..."