v1.1.1: air-gapped Slurm support, dual-mode runtime, GRPO multi-env hardening

.github/workflows/unit-tests.yml

@@ -41,4 +41,7 @@ jobs:
 
       - name: Test
         run: |
-          .venv/bin/pytest tests/ -v --tb=short
+          # --no-sync mirrors the GitLab CI invocation and prevents `uv run`
+          # from implicitly auto-installing heavy core deps that we
+          # deliberately skipped above.
+          uv run --no-sync pytest tests/ -v --tb=short

.gitlab-ci.yml

@@ -59,7 +59,10 @@ test:
     - uv venv --python 3.12
     - uv pip install -e ".[dev]" --no-deps
     - uv pip install pytest pytest-cov pytest-xdist pytest-timeout PyYAML omegaconf rich
-    - uv run pytest tests/ -v --tb=short
+    # --no-sync prevents `uv run` from implicitly auto-installing heavy core
+    # deps (nemo-skills etc) that we deliberately skipped above.  Without it,
+    # tests would pass here for the wrong reason and fail on GitHub CI.
+    - uv run --no-sync pytest tests/ -v --tb=short
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $CI_COMMIT_BRANCH == "main"

README.md

@@ -123,9 +123,18 @@ pytest
 
 ## 🔧 Cluster Setup
 
-To run workflows on a Slurm cluster, you need to configure containers and cluster settings.
-
-> **See [INSTALL.md](INSTALL.md)** for complete cluster setup (containers, cluster configuration, verification).
+To run workflows on a Slurm cluster you need to: (1) build the four NVFlow
+container images from the Dockerfiles in [`dockerfiles/`](dockerfiles/),
+(2) convert them to `.sqsh` for Slurm, and (3) write a cluster config
+(`cluster_configs/my_cluster.yaml`). The containers are self-sufficient —
+all dependencies are pre-installed, so no runtime downloads are needed.
+
+> **See [INSTALL.md](INSTALL.md)** for the complete setup guide
+> (build, sanity-check, `.sqsh` conversion, model staging, cluster
+> configuration, and verification).
+>
+> **See [`dockerfiles/docker_instructions.md`](dockerfiles/docker_instructions.md)**
+> for the build / multi-arch / sanity-check reference.
 
 Once cluster setup is complete, set the config directory:
 

cluster_configs/containers.yaml

@@ -10,33 +10,47 @@
 #   # Edit my_containers.yaml with your registry paths
 #   sbatch --account=<acct> scripts/setup_containers.sh --config cluster_configs/my_containers.yaml ./containers
 #
+# Naming:
+#   YAML KEYS (nemo-skills, nemo-rl, vllm, vllm-grpo, sglang) match the
+#   short names the workflow code uses to look up containers -- do NOT rename
+#   them.  Only the values (registry/tag refs) change between deployments.
+#
 # Format:
-#   - Simple string: image reference (e.g., your-registry/nemo-skills:latest)
+#   - Simple string: image reference (e.g., your-registry/nvflow-nemo-skills:0229040)
 #   - Nested object: Platform-specific tags (amd64/arm64 keys)
+#
+# Self-sufficient containers (all deps pre-installed, no runtime downloads):
+#   nemo-rl     -> dockerfiles/Dockerfile.nemo-rl     (base: nvcr.io/nvidia/nemo-rl:v0.6.0)
+#   nemo-skills -> dockerfiles/Dockerfile.nemo-skills (base: ubuntu:22.04, NeMo-Skills @ 0229040)
+#   vllm        -> dockerfiles/Dockerfile.vllm        (base: vllm/vllm-openai:v0.18.1)
+#   vllm-grpo   -> dockerfiles/Dockerfile.vllm-grpo   (base: vllm/vllm-openai:v0.17.1)
+#   sglang      -> pulled as-is from Docker Hub (no custom Dockerfile)
 
 containers:
   # ---------------------------------------------------------------------------
-  # Required: Build from NeMo-Skills Dockerfiles (see INSTALL.md Step 1a)
+  # Required: Built locally from dockerfiles/, then pushed to your registry
+  # (see INSTALL.md Step 1 and Step 2).
   # ---------------------------------------------------------------------------
-  # Tested: NeMo-Skills @ 0229040
-  nemo-skills: <YOUR_REGISTRY>/nemo-skills:<TAG>
+  # Tested: nvflow-nemo-rl:v0.6.0 (extends nvcr.io/nvidia/nemo-rl:v0.6.0)
+  nemo-rl: <YOUR_REGISTRY>/nvflow-nemo-rl:v0.6.0
+
+  # Tested: nvflow-nemo-skills:0229040 (NeMo-Skills @ commit 0229040)
+  nemo-skills: <YOUR_REGISTRY>/nvflow-nemo-skills:0229040
+
+  # Tested: nvflow-vllm:v0.18.1 (extends vllm/vllm-openai:v0.18.1; pre-cached
+  # tiktoken + openai_harmony; multi-arch amd64 + arm64)
+  vllm: <YOUR_REGISTRY>/nvflow-vllm:v0.18.1
+
+  # Tested: nvflow-vllm-grpo:v0.17.1 (extends vllm/vllm-openai:v0.17.1; pinned
+  # to match NeMo-RL v0.6.0 colocated vLLM)
+  vllm-grpo: <YOUR_REGISTRY>/nvflow-vllm-grpo:v0.17.1
 
   # ---------------------------------------------------------------------------
-  # Required: Pull pre-built from Docker Hub / NGC (no build needed)
+  # Required: Pulled as-is from Docker Hub (no custom Dockerfile)
   # ---------------------------------------------------------------------------
-  # Tested: vllm/vllm-openai:v0.18.1 (standalone SDG/eval)
-  vllm: <YOUR_REGISTRY>/nemo-skills-vllm:<TAG>
-  # Tested: vllm/vllm-openai:v0.17.1 (standalone GRPO rollouts/judge)
-  vllm-grpo: vllm/vllm-openai:v0.17.1
   # Tested: lmsysorg/sglang:v0.5.10.post1
   sglang: lmsysorg/sglang:v0.5.10.post1
 
-  # ---------------------------------------------------------------------------
-  # Required: Pull from NGC (see INSTALL.md Step 1b)
-  # ---------------------------------------------------------------------------
-  # Tested: nvcr.io/nvidia/nemo-rl:v0.6.0
-  nemo-rl: nvcr.io/nvidia/nemo-rl:v0.6.0
-
   # ---------------------------------------------------------------------------
   # Optional: Not currently used by NVFlow recipes
   # Uncomment and update if needed for your workflows.

cluster_configs/template-slurm.yaml

@@ -5,12 +5,12 @@
 #
 # Then update all <PLACEHOLDER> values with your settings.
 #
-# Container versions tested with this release:
-#   nemo-skills: NeMo-Skills @ 0229040
-#   vllm:        vllm/vllm-openai v0.18.1 (standalone SDG/eval)
-#   vllm-grpo:   vllm/vllm-openai v0.17.1 (standalone GRPO rollouts/judge)
+# Container versions tested with this release (self-sufficient, no runtime downloads):
+#   nemo-skills: nvflow-nemo-skills (NeMo-Skills @ 0229040)
+#   vllm:        nvflow-vllm (base vllm/vllm-openai v0.18.1, standalone SDG/eval)
+#   vllm-grpo:   nvflow-vllm-grpo (base vllm/vllm-openai v0.17.1, GRPO rollouts/judge)
 #   sglang:      lmsysorg/sglang v0.5.10.post1
-#   nemo-rl:     nvcr.io/nvidia/nemo-rl:v0.6.0 (includes vLLM 0.17.1 colocated)
+#   nemo-rl:     nvflow-nemo-rl (base nvcr.io/nvidia/nemo-rl:v0.6.0, pre-built venvs + Gym)
 #
 # Reference: https://github.com/NVIDIA/NeMo-Skills
 
@@ -79,12 +79,12 @@ extra_sandbox_args:
 # =============================================================================
 # After converting containers to .sqsh format (see INSTALL.md), paste paths here.
 containers:
-  # Required containers
-  nemo-skills: <PATH_TO_CONTAINERS>/nemo-skills.sqsh   # Orchestration client (eval, SDG, data prep)
-  vllm: <PATH_TO_CONTAINERS>/vllm.sqsh                 # vLLM v0.18.1 standalone (SDG, eval)
-  vllm-grpo: <PATH_TO_CONTAINERS>/vllm-grpo.sqsh       # vLLM v0.17.1 standalone (GRPO rollouts, judge)
-  sglang: <PATH_TO_CONTAINERS>/sglang.sqsh             # sglang inference server (SDG stages 3-5)
-  nemo-rl: <PATH_TO_CONTAINERS>/nemo-rl.sqsh           # NeMo-RL v0.6.0 for SFT and GRPO training
+  # Required containers (self-sufficient — all deps pre-installed, no runtime downloads)
+  nemo-skills: <PATH_TO_CONTAINERS>/nvflow-nemo-skills.sqsh    # Orchestration client (eval, SDG, data prep)
+  vllm: <PATH_TO_CONTAINERS>/nvflow-vllm.sqsh                  # vLLM v0.18.1 standalone (SDG, eval)
+  vllm-grpo: <PATH_TO_CONTAINERS>/nvflow-vllm-grpo.sqsh        # vLLM v0.17.1 standalone (GRPO rollouts, judge)
+  sglang: <PATH_TO_CONTAINERS>/sglang.sqsh                     # sglang inference server (SDG stages 3-5)
+  nemo-rl: <PATH_TO_CONTAINERS>/nvflow-nemo-rl.sqsh            # NeMo-RL v0.6.0 for SFT and GRPO training
   # Optional containers (not currently used by NVFlow recipes)
   # trtllm: <PATH_TO_CONTAINERS>/trtllm.sqsh
   # megatron: <PATH_TO_CONTAINERS>/megatron.sqsh
@@ -98,14 +98,16 @@ containers:
 mounts:
   - <CLUSTER_PATH_TO_HF_MODELS>:/hf_models   # HuggingFace models
   - <CLUSTER_PATH_TO_WORKSPACE>:/workspace   # Your workspace
-  # --- GRPO / RL Training (required for collect_rollouts and training stages) ---
-  # NeMo-RL source mount: overlays the container's built-in /opt/NeMo-RL.
-  - <PATH_TO_NEMO_RL_CLONE>:/opt/NeMo-RL
-  # Gym overlay: mount your NeMo-Gym clone inside the NeMo-RL source tree.
-  # Harmless for SFT/SDG/eval -- only accessed by GRPO stages.
-  - <PATH_TO_GYM_CLONE>:/opt/NeMo-RL/3rdparty/Gym-workspace/Gym
   # Add more mounts as needed:
   # - /lustre/data:/data
+  #
+  # --- NeMo-RL / NeMo-Gym source overlays (dev mode only) ---
+  # The nvflow-nemo-rl container includes NeMo-RL source and pre-built Gym
+  # venvs. Mounting host clones here shadows the container's venvs and
+  # breaks GRPO stages. Only uncomment for local source iteration with
+  # NRL_FORCE_REBUILD_VENVS=true enabled below.
+  # - <PATH_TO_NEMO_RL_CLONE>:/opt/NeMo-RL
+  # - <PATH_TO_GYM_CLONE>:/opt/NeMo-RL/3rdparty/Gym-workspace/Gym
 
 # =============================================================================
 # Timeouts (per partition)
@@ -131,10 +133,30 @@ env_vars:
   - TOKENIZERS_PARALLELISM=false  # Disable HF tokenizer Rayon threads (prevents vLLM RefCell race)
   - VIRTUAL_ENV=  # Unset to prevent host venv from interfering with container
   - VIRTUAL_ENV_PROMPT=  # Unset venv prompt
-  # Rebuild Ray venvs when NeMo-RL source mount changes (e.g., new branch/commit).
-  # Without this, workers reuse stale cached venvs and may fail with import errors.
-  # Safe to leave enabled — only rebuilds when the source tree actually changes.
+
+  # --- Offline enforcement (recommended) ---
+  # Prevents accidental network calls at runtime. Containers are self-sufficient.
+  # Clear HF_*_OFFLINE temporarily for one-time stages that download external
+  # data (download_sec_filings, create_seed_data). UV_OFFLINE should stay set.
+  - HF_HUB_OFFLINE=1
+  - HF_DATASETS_OFFLINE=1
+  - TRANSFORMERS_OFFLINE=1
+  - UV_OFFLINE=true
+
+  # Pre-cached tiktoken encodings (baked into vllm/vllm-grpo containers;
+  # set here for uniform coverage across all container types).
+  - TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache
+  - TIKTOKEN_RS_CACHE_DIR=/opt/tiktoken_cache
+  - TIKTOKEN_ENCODINGS_BASE=/opt/tiktoken_cache
+
+  # Ray worker port range — avoids collisions with OS ephemeral ports.
+  - MIN_WORKER_PORT=7000
+  - MAX_WORKER_PORT=8000
+
+  # Only enable in dev mode when iterating on NeMo-RL/Gym source overlays.
+  # Forces Ray workers to rebuild venvs from mounted source (requires internet).
   # - NRL_FORCE_REBUILD_VENVS=true
+
   # API keys (keep these secret, don't commit to git!)
   # - HF_TOKEN=<YOUR_HF_TOKEN>
   # - WANDB_API_KEY=<YOUR_WANDB_KEY>

dockerfiles/Dockerfile.nemo-rl

@@ -0,0 +1,131 @@
+# =============================================================================
+# NVFlow NeMo-RL Container
+# =============================================================================
+# Extends the NeMo-RL nightly container with NeMo-Skills and the NeMo-Gym
+# finance agent for NVFlow workflows.  The base image ships with frozen
+# environments and pre-built Ray venvs; this Dockerfile adds the NeMo-Skills
+# package, replaces the Gym submodule with a feature branch that includes the
+# finance-SEC-search resource server and finance agent, pre-builds all Gym
+# component venvs, and relocates paths for Slurm/enroot compatibility.
+#
+# Build:
+#   docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:latest .
+# =============================================================================
+
+ARG BASE_IMAGE=nvcr.io/nvidia/nemo-rl:v0.6.0
+FROM ${BASE_IMAGE}
+
+# --- Symlink for NeMo-Skills code that references /opt/NeMo-RL (wrong case) --
+RUN ln -sf /opt/nemo-rl /opt/NeMo-RL
+
+# --- Upgrade uv (nemo-gym may require newer features than what the nightly ships)
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# --- Pre-cache Python interpreter for uv (air-gapped safety net) -------------
+# Gym's cli_setup_command runs `uv venv --python <version>`.  If
+# skip_venv_if_present ever misses, uv still needs a local interpreter.
+RUN /root/.local/bin/uv python install 3.12
+
+# --- Install NeMo-Skills into the frozen venv --------------------------------
+ARG NEMO_SKILLS_COMMIT=022904023ad7a83a87662a313cf72e7df5891d55
+RUN git clone https://github.com/NVIDIA-NeMo/Skills.git /opt/NeMo-Skills && \
+    cd /opt/NeMo-Skills && git checkout ${NEMO_SKILLS_COMMIT} && \
+    /root/.local/bin/uv pip install --python /opt/nemo_rl_venv/bin/python .
+
+# --- Replace NeMo-Gym submodule with feature branch -------------------------
+# The feature branch includes the finance-SEC-search resource server and
+# finance agent that are not yet on main.
+ARG NEMO_GYM_BRANCH=ude/finance-sec-search-v2
+RUN rm -rf /opt/nemo-rl/3rdparty/Gym-workspace/Gym && \
+    git clone --branch ${NEMO_GYM_BRANCH} \
+        https://github.com/NVIDIA-NeMo/Gym.git \
+        /opt/nemo-rl/3rdparty/Gym-workspace/Gym
+
+# --- Pre-build Gym venv ------------------------------------------------------
+WORKDIR /opt/nemo-rl/3rdparty/Gym-workspace/Gym
+RUN /root/.local/bin/uv venv .venv --python 3.12 && \
+    . .venv/bin/activate && \
+    /root/.local/bin/uv sync --active --extra dev
+
+# Install finance-specific dependencies into Gym venv
+# uvicorn>=0.37.0 is required for timeout_worker_healthcheck support;
+# uv sync resolves from the parent nemo-rl workspace lock (0.35.0) instead
+# of the Gym lock, so we force the correct version here.
+RUN . .venv/bin/activate && \
+    /root/.local/bin/uv pip install aiohttp beautifulsoup4 "tavily==1.1.0" tenacity "uvicorn>=0.37.0"
+
+# --- Symlink component venvs to the main Gym venv ---------------------------
+# Each NeMo-Gym component expects its own .venv/; symlinking avoids multi-GB
+# duplication and guarantees every component runs with the same packages.
+RUN for component in \
+      resources_servers/equivalence_llm_judge \
+      resources_servers/finance_sec_search \
+      responses_api_agents/simple_agent \
+      responses_api_agents/finance_agent \
+      responses_api_models/openai_model \
+      responses_api_models/vllm_model; do \
+        dir="/opt/nemo-rl/3rdparty/Gym-workspace/Gym/$component"; \
+        [ -d "$dir" ] && ln -sf /opt/nemo-rl/3rdparty/Gym-workspace/Gym/.venv "$dir/.venv"; \
+    done
+
+WORKDIR /
+
+# --- Install Gym into the NemoGym Ray venv ------------------------------------
+# The pre-built Ray venv from the base image is stale (built from the old Gym
+# submodule). Install the new Gym branch editable + all deps so the Ray actor
+# can import nemo_gym without missing modules (e.g. gprof2dot, pydot).
+RUN /root/.local/bin/uv pip install \
+      --python /opt/ray_venvs/nemo_rl.environments.nemo_gym.NemoGym/bin/python \
+      -e /opt/nemo-rl/3rdparty/Gym-workspace/Gym
+
+# --- Align numpy across all Ray venvs to match the main venv ----------------
+# NeMo-Skills may upgrade numpy; mismatched versions cause pickle failures
+# when Ray serializes data between the main process and worker processes.
+RUN MAIN_NP=$(/opt/nemo_rl_venv/bin/python -c "import numpy; print(numpy.__version__)") && \
+    for venv in /opt/ray_venvs/*/; do \
+      "$venv/bin/pip" install --no-cache-dir "numpy==$MAIN_NP" 2>/dev/null || true; \
+    done
+
+# --- Relocate /root/.local/ → /opt/ -----------------------------------------
+# enroot/pyxis on Slurm mounts the user's home directory over /root at runtime,
+# which shadows everything uv installed there during the Docker build.
+# NOTE: Do NOT move /root/.cache/uv — base-image venvs symlink into it.
+RUN REAL_PYTHON=$(readlink /opt/nemo_rl_venv/bin/python) && \
+    mv /root/.local/share/uv/python /opt/uv-python && \
+    find /opt/uv-python -maxdepth 1 -type l | while read link; do \
+      target=$(readlink "$link") && \
+      new_target=$(echo "$target" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \
+      ln -sf "$new_target" "$link"; \
+    done && \
+    NEW_PYTHON=$(echo "$REAL_PYTHON" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \
+    ln -sf "$NEW_PYTHON" /opt/nemo_rl_venv/bin/python && \
+    sed -i "s|/root/.local/share/uv/python|/opt/uv-python|g" /opt/nemo_rl_venv/pyvenv.cfg && \
+    mv /root/.local/bin /opt/uv-bin
+
+# --- Fix pre-built Ray venvs (same /root/ relocation) -----------------------
+RUN for cfg in /opt/ray_venvs/*/pyvenv.cfg; do \
+      sed -i "s|/root/.local/share/uv/python|/opt/uv-python|g" "$cfg"; \
+    done && \
+    find /opt/ray_venvs/ -type l | while read link; do \
+      target=$(readlink "$link") && \
+      case "$target" in */root/.local/share/uv/python*) \
+        new_target=$(echo "$target" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \
+        ln -sf "$new_target" "$link" ;; \
+      esac; \
+    done
+
+# --- Fix Gym venv (same /root/ relocation) ----------------------------------
+RUN GYM_VENV=/opt/nemo-rl/3rdparty/Gym-workspace/Gym/.venv && \
+    sed -i "s|/root/.local/share/uv/python|/opt/uv-python|g" "$GYM_VENV/pyvenv.cfg" && \
+    find "$GYM_VENV" -type l | while read link; do \
+      target=$(readlink "$link") && \
+      case "$target" in */root/.local/share/uv/python*) \
+        new_target=$(echo "$target" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \
+        ln -sf "$new_target" "$link" ;; \
+      esac; \
+    done
+
+# --- Runtime environment -----------------------------------------------------
+ENV VIRTUAL_ENV=/opt/nemo_rl_venv
+ENV PATH=/opt/uv-bin:/opt/nemo_rl_venv/bin:$PATH
+ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python