From 9182e5c0d3587be6ca0a6090242ebdca141f6af1 Mon Sep 17 00:00:00 2001 From: Pritam Gundecha Date: Tue, 17 Mar 2026 16:54:11 -0700 Subject: [PATCH 1/2] Housekeeping: fix edgar version, doc cleanup, and enable unit tests Signed-off-by: Pritam Gundecha --- .github/workflows/unit-tests.yml | 7 ++-- .gitlab-ci.yml | 38 +++++++++---------- README.md | 2 +- docs/architecture/ARCHITECTURE_INDEX.md | 2 +- docs/architecture/DIAGRAMS_SUMMARY.md | 2 +- docs/cluster-configuration.md | 1 - docs/diagrams/README.md | 4 +- docs/recipes/finance/README.md | 2 +- docs/recipes/finance/troubleshooting.md | 6 +-- .../finance/workflows/01-download-sec.md | 26 ++----------- .../workflows/download_sec_filings.yaml | 4 +- 11 files changed, 36 insertions(+), 58 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 23c3137..55a7fb5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -33,10 +33,11 @@ jobs: - name: Setup for test run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends git curl ca-certificates curl -LsSf https://astral.sh/uv/install.sh | sh - uv sync --python 3.12 --no-install-project --extra dev + uv venv --python 3.12 + uv pip install pytest pytest-cov pytest-timeout + uv pip install PyYAML omegaconf rich + uv pip install -e . --no-deps - name: Test run: | diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d39c7b0..8772866 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ default: stages: - lint - # - test # Uncomment when tests are ready + - test - build - publish @@ -55,25 +55,25 @@ dco-check: - if: $CI_PIPELINE_SOURCE == "merge_request_event" # ============================================================================= -# TEST STAGE (disabled for now) +# TEST STAGE # ============================================================================= -# test: -# stage: test -# image: python:3.12-slim -# before_script: -# - pip install uv -# - uv sync --all-extras -# script: -# - uv run pytest tests/ -v --tb=short -# rules: -# # Run on merge requests -# - if: $CI_PIPELINE_SOURCE == "merge_request_event" -# # Run on main branch -# - if: $CI_COMMIT_BRANCH == "main" -# # Run on dev/ci-test branch (for testing) -# - if: $CI_COMMIT_BRANCH == "dev/ci-test" -# # Run on tags -# - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/ +test: + stage: test + image: python:3.12-slim + before_script: + - apt-get update && apt-get install -y git + - pip install uv==0.9.22 + - uv venv --python 3.12 + - uv pip install pytest pytest-cov pytest-timeout + - uv pip install PyYAML omegaconf rich + - uv pip install -e . --no-deps + script: + - uv run pytest tests/ -v --tb=short + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_BRANCH == "dev" + - if: $CI_COMMIT_BRANCH =~ /^feature\// # ============================================================================= # BUILD STAGE diff --git a/README.md b/README.md index 1efd6ba..e32bf90 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ uv --version # Should show: uv 0.9.22 ## 📦 Installation ```bash -git clone ssh://git@gitlab-master.nvidia.com:12051/swdl-nemollm-mlops/alignment-data/nvflow.git +git clone https://github.com/NVIDIA/nvflow.git cd nvflow # For users diff --git a/docs/architecture/ARCHITECTURE_INDEX.md b/docs/architecture/ARCHITECTURE_INDEX.md index b23cd25..ef72c5e 100644 --- a/docs/architecture/ARCHITECTURE_INDEX.md +++ b/docs/architecture/ARCHITECTURE_INDEX.md @@ -137,7 +137,7 @@ The NVFlow architecture is documented across multiple files, each serving a spec - Online: https://mermaid.live/ - VS Code: Mermaid Preview extension - CLI: `mmdc -i diagram.mmd -o diagram.png` -- GitHub/GitLab: Native rendering +- GitHub: Native rendering **When to Use:** - Need visual understanding diff --git a/docs/architecture/DIAGRAMS_SUMMARY.md b/docs/architecture/DIAGRAMS_SUMMARY.md index 0ca5775..a265c18 100644 --- a/docs/architecture/DIAGRAMS_SUMMARY.md +++ b/docs/architecture/DIAGRAMS_SUMMARY.md @@ -193,7 +193,7 @@ Use Case: Cluster setup and deployment planning ## 🎨 Diagram Rendering Examples -### In Markdown (GitLab/GitHub) +### In Markdown (GitHub) ````markdown ```mermaid graph TB diff --git a/docs/cluster-configuration.md b/docs/cluster-configuration.md index 20795d3..4b52996 100644 --- a/docs/cluster-configuration.md +++ b/docs/cluster-configuration.md @@ -322,7 +322,6 @@ env_vars: | `OPENAI_API_KEY` | OpenAI API access | [OpenAI Dashboard](https://platform.openai.com/api-keys) | | `HF_TOKEN` | Hugging Face model downloads | [HF Settings](https://huggingface.co/settings/tokens) | | `WANDB_API_KEY` | Weights & Biases logging | [W&B Settings](https://wandb.ai/authorize) | -| `GITLAB_TOKEN` | GitLab API access | Your GitLab instance | **Security notes:** - ⚠️ **Never commit API keys to git** (config is gitignored) diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md index 6e18351..e8dece4 100644 --- a/docs/diagrams/README.md +++ b/docs/diagrams/README.md @@ -85,8 +85,8 @@ mmdc -i deployment-architecture.mmd -o deployment-architecture.png mmdc -i architecture-overview.mmd -o architecture-overview.svg ``` -### Option 4: GitHub/GitLab Rendering -GitLab and GitHub now support Mermaid diagrams natively in Markdown files. You can include them in documentation using: +### Option 4: GitHub Rendering +GitHub supports Mermaid diagrams natively in Markdown files. You can include them in documentation using: ````markdown ```mermaid diff --git a/docs/recipes/finance/README.md b/docs/recipes/finance/README.md index d3fcafa..af98cb7 100644 --- a/docs/recipes/finance/README.md +++ b/docs/recipes/finance/README.md @@ -200,4 +200,4 @@ uv run nflow run-all --config nvflow/recipes/finance/workflows/grpo/qwen3_4b.yam - 📖 **[Workflow Guides](workflows/)** - How to run each workflow - 🔧 **[Stage Reference](stages/)** - Technical specifications - 🚨 **[Troubleshooting Guide](troubleshooting.md)** - Solutions to common issues -- 🐛 **GitLab Issues** - Report bugs or request features +- 🐛 **[GitHub Issues](https://github.com/NVIDIA/nvflow/issues)** - Report bugs or request features diff --git a/docs/recipes/finance/troubleshooting.md b/docs/recipes/finance/troubleshooting.md index e353f55..e6da1aa 100644 --- a/docs/recipes/finance/troubleshooting.md +++ b/docs/recipes/finance/troubleshooting.md @@ -64,7 +64,6 @@ vim $NEMO_SKILLS_CONFIG_DIR/my_cluster.yaml - Verify cluster has internet access - Check firewall rules for SEC EDGAR (sec.gov) - Ensure model repositories are accessible -- Check GitLab token: `echo $GITLAB_TOKEN` ### Ray Cluster Initialization Issues (SLURM 25.11.2+) @@ -321,7 +320,6 @@ ls outputs/.../checkpoints/final/ **Common issues:** - Rate limiting: See [SEC Download Rate Limits](#sec-download-rate-limits) -- Authentication: Ensure GITLAB_TOKEN is exported - Invalid tickers: Verify company symbols in config **Quick check:** @@ -439,7 +437,7 @@ If issues persist after trying these solutions: - [Stage documentation](stages/) 3. **Report issues:** - - Check existing GitLab issues + - Check existing [GitHub issues](https://github.com/NVIDIA/nvflow/issues) - Create new issue with: - Workflow and stage name - Error message (full traceback) @@ -447,5 +445,5 @@ If issues persist after trying these solutions: - Environment details 4. **Feature requests:** - - Submit GitLab issue with `[Feature Request]` tag + - Submit [GitHub issue](https://github.com/NVIDIA/nvflow/issues) with `[Feature Request]` tag - Describe use case and expected behavior diff --git a/docs/recipes/finance/workflows/01-download-sec.md b/docs/recipes/finance/workflows/01-download-sec.md index 5e64526..3325129 100644 --- a/docs/recipes/finance/workflows/01-download-sec.md +++ b/docs/recipes/finance/workflows/01-download-sec.md @@ -15,12 +15,6 @@ Download SEC 10-K, 10-Q, and 8-K filings from the EDGAR database for specified c Before running this workflow, ensure you have: - ✅ **Output directory writable** (workflow creates `outputs/finance/demo/workflow-2-download-sec/step-0-download`) -- ✅ **GitLab token exported** (for internal NVIDIA deployments): - ```bash - export GITLAB_TOKEN= - ``` - - **Why needed:** This workflow pip installs the `sec-downloader-parser` tool from an internal GitLab repository during execution - - **For external deployments:** This dependency can be pre-installed or replaced with an alternative SEC downloader - ✅ **SEC EDGAR identity configured** in your workflow YAML: ```yaml @@ -39,15 +33,14 @@ This workflow has 2 steps: 1. **NVFlow Stage** (`nvflow/recipes/finance/stages/download/download_sec_filings.py`) - Orchestrates the download job on your Slurm cluster - Manages containerized execution and output paths - - Pip installs the SEC downloader tool at runtime -2. **SEC Downloader Tool** (external Python package from GitLab) +2. **SEC Downloader Tool** (bundled in NVFlow) - Interfaces directly with the SEC EDGAR API - Downloads filings in HTML format - Parses and extracts individual sections (Items 1, 1A, 2, etc.) from 10-K/10-Q filings - Generates metadata file (`sec_metadata.parquet`) for downstream stages -The NVFlow stage wraps the external tool, handling cluster integration and output management while the tool itself handles SEC API interaction and parsing. +The NVFlow stage handles cluster integration and output management while the bundled SEC downloader tool handles SEC API interaction and parsing. ## Configuration Files @@ -86,9 +79,7 @@ The NVFlow stage wraps the external tool, handling cluster integration and outpu ## Stage: sap-500 -Downloads SEC filings using the `sec-downloader-parser` tool ([internal NVIDIA GitLab](https://gitlab-master.nvidia.com/swdl-nemollm-mlops/alignment-data/sec-downloader-parser)). - -> **Note for external deployments:** The sec-downloader-parser is currently hosted on internal NVIDIA infrastructure. For external use, you can either pre-install the tool or replace it with an alternative SEC EDGAR downloader that produces the same output format. +Downloads SEC filings using the bundled `sec-downloader-parser` tool. **See [technical reference](../stages/download-sec.md) for detailed parameters and configuration.** @@ -193,17 +184,6 @@ Both SDG workflows consume the downloaded filings. - There was a network delay in downloading the file so the read timed out - Wait a few minutes and retry -### Missing GitLab Token (Internal NVIDIA Only) - -**Symptom:** pip install fails for sec-downloader-parser - -**Solution:** -```bash -export GITLAB_TOKEN= -``` - -**Note:** See Prerequisites section above for more details on GitLab token requirements and external deployment alternatives. - ### Storage Space **Symptom:** Disk full during download diff --git a/nvflow/recipes/finance/workflows/download_sec_filings.yaml b/nvflow/recipes/finance/workflows/download_sec_filings.yaml index b8584d8..3ab52c8 100644 --- a/nvflow/recipes/finance/workflows/download_sec_filings.yaml +++ b/nvflow/recipes/finance/workflows/download_sec_filings.yaml @@ -28,7 +28,7 @@ stages: stage_kwargs: installation_command: >- pip install -q --root-user-action=ignore - edgartools sec-parser pandas pyarrow tqdm httpx tzdata && + edgartools==5.20.2 sec-parser pandas pyarrow tqdm httpx tzdata && ln -sfn $(python -c "import tzdata; print(tzdata.__path__[0])")/zoneinfo /usr/share/zoneinfo 2>/dev/null || true @@ -42,6 +42,6 @@ stages: stage_kwargs: installation_command: >- pip install -q --root-user-action=ignore - edgartools sec-parser pandas pyarrow tqdm httpx tzdata && + edgartools==5.20.2 sec-parser pandas pyarrow tqdm httpx tzdata && ln -sfn $(python -c "import tzdata; print(tzdata.__path__[0])")/zoneinfo /usr/share/zoneinfo 2>/dev/null || true From 8fe12684c5e701690c2ce2b07dd8c0312839c857 Mon Sep 17 00:00:00 2001 From: Pritam Gundecha Date: Tue, 17 Mar 2026 17:55:06 -0700 Subject: [PATCH 2/2] Fix CI: use .venv/bin/pytest to avoid uv resolving full dependency tree Signed-off-by: Pritam Gundecha --- .github/workflows/unit-tests.yml | 2 +- .gitlab-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 55a7fb5..79be005 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,4 +41,4 @@ jobs: - name: Test run: | - uv run pytest tests/ -v --tb=short + .venv/bin/pytest tests/ -v --tb=short diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8772866..89951cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,7 @@ test: - uv pip install PyYAML omegaconf rich - uv pip install -e . --no-deps script: - - uv run pytest tests/ -v --tb=short + - .venv/bin/pytest tests/ -v --tb=short rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH == "main"