From 075830b1d32125d452292f51bf6db107d2b16d41 Mon Sep 17 00:00:00 2001 From: Pritam Gundecha Date: Tue, 4 Aug 2026 17:25:48 -0700 Subject: [PATCH] v1.1.2: air-gapped NeMo-RL v0.7.0 stack, multimodal SDG, validated finance tool-calling recipe Air-gapped stack on NeMo-RL v0.7.0. The trainer image is now built locally with one Gym virtual environment baked per component, so GRPO and SFT training needs no Gym source mount and resolves no packages at job time. Only sglang is still pulled as-is, and every image now ships for both amd64 and arm64. nvflow-client launcher image. A new container that submits and monitors workflows from outside the cluster, with air-gapped install scripts and tunnel-aware launching for collect_rollouts, training, and verify. Finance tool-calling recipe. The multi-turn SEC-search recipe is validated end to end on the new stack: SFT, GRPO, and evaluation all reproduce the prior baseline with no regression. Finance RL / GRPO. Multi-environment GRPO with per-environment judges and per-environment eval output directories. Gym-only stages, collect_rollouts and compute_rewards, now run on a CPU-only container instead of holding GPUs. Adds Nemotron Nano 30B and Qwen3-30B configs, and carries train and validation histograms into the consolidated W&B dashboard. Document-grounded SDG. Generalized domain interface so a new domain plugs in without forking the recipe, plus follow-up fixes. HopChain multimodal SDG recipe for open-source release. Local judge support for evaluation, so the demo recipes no longer need a hosted judge endpoint. Tooling and tests. A dependency-free rollout-trace viewer, and the unit-test suite grows from 2 modules to 16 covering rollout, verify, dataset, and JSONL utilities. Documentation accuracy pass. All markdown reconciled against the shipped behaviour, and the third-party license notice synced to the lockfile. Dependency and CVE floors raised across the images. Signed-off-by: Pritam Gundecha --- .dockerignore | 41 + .github/workflows/secrets-detector.yml | 7 +- .github/workflows/unit-tests.yml | 5 +- .gitignore | 10 +- .gitlab-ci.yml | 30 +- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 11 +- INSTALL.md | 353 +-- LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md | 85 +- NOTICE | 2 +- README.md | 62 +- cluster_configs/containers.yaml | 65 +- cluster_configs/template-slurm.yaml | 62 +- dockerfiles/Dockerfile.nemo-gym | 102 + dockerfiles/Dockerfile.nemo-rl | 240 +- dockerfiles/Dockerfile.nemo-skills | 207 +- dockerfiles/Dockerfile.nvflow | 151 ++ dockerfiles/Dockerfile.vllm | 65 +- dockerfiles/Dockerfile.vllm-grpo | 55 - dockerfiles/README.md | 58 +- dockerfiles/docker_instructions.md | 276 ++- docs/{architecture => }/ARCHITECTURE.md | 119 +- docs/architecture/ARCHITECTURE_INDEX.md | 435 ---- .../ARCHITECTURE_QUICK_REFERENCE.md | 276 --- docs/architecture/DIAGRAMS_SUMMARY.md | 310 --- docs/cluster-configuration.md | 52 +- docs/development/nemo-rl-gym.md | 45 + .../sdg/document_grounded/ADDING_A_DOMAIN.md | 714 ++++++ .../dgsdg-add-new-domain.png | Bin 0 -> 298475 bytes docs/diagrams/README.md | 7 +- docs/diagrams/finance-pipeline.mmd | 16 +- docs/maintainers/containers.md | 137 ++ docs/recipes/finance/README.md | 34 +- docs/recipes/finance/quick-start.md | 102 +- .../finance/stages/document-grounded-sdg.md | 368 ++-- docs/recipes/finance/stages/download-sec.md | 7 +- docs/recipes/finance/stages/eval.md | 6 +- .../finance/stages/finance-agent-eval.md | 34 +- docs/recipes/finance/stages/grpo.md | 170 +- docs/recipes/finance/stages/sft.md | 91 +- docs/recipes/finance/troubleshooting.md | 72 +- .../workflows/02-template-based-sdg.md | 1 + .../workflows/03-document-grounded-sdg.md | 234 +- docs/recipes/finance/workflows/04-sft.md | 8 +- docs/recipes/finance/workflows/05-eval.md | 16 +- .../workflows/06-finance-agent-eval.md | 64 - docs/recipes/finance/workflows/06-grpo.md | 20 +- docs/recipes/multimodal/README.md | 112 + docs/recipes/multimodal/quick-start.md | 326 +++ docs/remote-launch.md | 152 ++ docs/trace-viewer.md | 61 + nvflow/core/__init__.py | 20 +- nvflow/core/workflow_runner.py | 5 +- nvflow/generic_stage/__init__.py | 15 + nvflow/generic_stage/sdg/__init__.py | 15 + .../sdg/document_grounded/__init__.py | 49 + .../sdg/document_grounded/_helpers.py | 365 ++++ .../sdg/document_grounded/_schemas.py | 181 ++ .../sdg/document_grounded/_trim_cli.py | 121 + .../document_grounded}/aggregate_answers.py | 39 +- .../document_grounded}/dg_sdg_preprocess.py | 82 +- .../document_grounded/dgsdg_post_process.py} | 53 +- .../sdg/document_grounded/evaluate_answers.py | 152 ++ .../sdg/document_grounded/generate_answers.py | 195 ++ .../generate_verified_questions.py | 252 +++ .../gym_genselect_answers.py | 155 ++ nvflow/lib/cli_cmd.py | 165 ++ nvflow/lib/rl/helpers.py | 213 +- nvflow/lib/rl/rollout.py | 692 ++++-- nvflow/lib/rl/verify.py | 93 +- nvflow/lib/runtime.py | 4 +- nvflow/lib/sdg/__init__.py | 25 + nvflow/lib/sdg/document_grounded/__init__.py | 33 + nvflow/lib/sdg/document_grounded/aggregate.py | 293 +++ .../sdg/document_grounded/analyze_rollouts.py | 109 + nvflow/lib/sdg/document_grounded/chunking.py | 524 +++++ .../sdg/document_grounded/enrich_rollouts.py | 282 +++ .../enrich_rollouts_evaluate.py | 50 + nvflow/lib/sdg/document_grounded/evaluate.py | 265 +++ nvflow/lib/sdg/document_grounded/genselect.py | 389 ++++ .../gym_configs/sdg_equivalence_judge.yaml | 55 + .../gym_configs/sdg_format_verification.yaml | 58 + nvflow/lib/sdg/document_grounded/metadata.py | 49 + .../lib/sdg/document_grounded/postprocess.py | 199 ++ .../sdg/document_grounded/preprocess.py} | 465 ++-- .../sdg/document_grounded/responses_api.py | 630 ++++++ nvflow/lib/sdg/document_grounded/sampling.py | 73 + nvflow/lib/vllm_compat.py | 123 -- .../recipes/example/workflows/sdg_simple.yaml | 2 +- .../recipes/finance/datasets/finance_judge.py | 2 +- .../document_grounded_generate_questions.yaml | 39 +- .../prompts/finance_agent_instructions.yaml | 2 +- .../prompts/finance_openqa_judge_overlay.yaml | 4 +- .../finance_sec_search_template_with_web.yaml | 24 +- ...nance_sec_search_template_without_web.yaml | 16 +- .../finance/prompts/judge_difficulty.yaml | 31 - .../finance/stages/evaluation/evaluate.py | 33 +- .../finance/stages/evaluation/prepare_data.py | 4 +- .../stages/rl/apply_prompt_template.py | 44 +- .../stages/rl/convert_to_responses_api.py | 10 +- .../finance/stages/rl/prefetch_cache.py | 16 +- .../recipes/finance/stages/rl/prepare_data.py | 19 +- nvflow/recipes/finance/stages/rl/training.py | 76 +- .../finance/stages/rl/validate_questions.py | 76 +- nvflow/recipes/finance/stages/sdg/__init__.py | 22 +- .../stages/sdg/difficulty_estimation.py | 212 -- ...ded_question_answer_generation_pipeline.py | 237 -- .../finance/stages/sdg/evaluate_answers.py | 133 -- .../finance/stages/sdg/filter_answers.py | 6 +- .../finance/stages/sdg/generate_answers.py | 6 +- .../finance/stages/sdg/generate_questions.py | 6 +- .../finance/stages/sdg/genselect_answers.py | 10 +- nvflow/recipes/finance/stages/sft/training.py | 11 +- .../stages/shared/data_transformation.py | 54 +- .../utils/evaluation/checkpoint_converter.py | 9 +- .../evaluation/prepare_benchmark_data.py | 14 +- .../finance/utils/rl/aggregate_seeds.py | 38 +- .../finance/utils/rl/apply_validate_filter.py | 214 +- .../finance/utils/rl/enrich_rollouts.py | 59 +- .../finance/utils/rl/filter_training_data.py | 36 +- .../utils/rl/parse_validate_responses.py | 37 +- .../finance/utils/rl/postprocess_validate.py | 165 ++ .../utils/rl/prompt_template_applier.py | 112 +- .../utils/rl/regex_prefilter_questions.py | 425 ++-- .../utils/rl/responses_api_converter.py | 150 +- .../recipes/finance/utils/rl/shuffle_jsonl.py | 51 +- .../utils/sdg/dg_sdg_data_preprocess.py | 133 +- .../finance/utils/sdg/dgsdg_post_process.py | 228 -- .../utils/sdg/difficulty_estimation.py | 365 ---- .../finance/utils/sdg/sec_callbacks.py | 89 + .../finance/utils/sdg/sec_postprocess.py | 54 + .../finance/utils/sdg/sec_question_prep.py | 39 + .../utils/shared/dataset_transformer.py | 113 +- .../recipes/finance/workflows/eval/base.yaml | 8 +- .../finance/workflows/eval/baselines.yaml | 8 +- .../recipes/finance/workflows/eval/demo.yaml | 12 +- .../recipes/finance/workflows/eval/smoke.yaml | 4 +- .../recipes/finance/workflows/grpo/base.yaml | 110 +- .../finance/workflows/grpo/grpo_presets.yaml | 1 + .../workflows/grpo/nemotron_nano_30b_a3b.yaml | 489 +++++ .../grpo/nemotron_nano_30b_a3b_b1.yaml | 413 ++++ .../grpo/nemotron_nano_30b_a3b_b2.yaml | 390 ++++ .../grpo/nemotron_nano_30b_a3b_b3.yaml | 390 ++++ .../grpo/nemotron_nano_30b_a3b_b4.yaml | 390 ++++ .../grpo/nemotron_nano_30b_a3b_b5.yaml | 388 ++++ .../grpo/nemotron_nano_30b_a3b_smoke1.yaml | 481 ++++ .../grpo/nemotron_nano_30b_a3b_smoke2.yaml | 465 ++++ .../overlays/finance_sec_search_demo.yaml | 4 + .../grpo/overlays/finance_sec_search_env.yaml | 21 +- .../overlays/finance_sec_search_sap500.yaml | 2 + .../finance/workflows/grpo/qwen3_30b_a3b.yaml | 84 +- .../finance/workflows/grpo/qwen3_4b.yaml | 31 +- .../workflows/grpo/qwen3_4b_finsec.yaml | 42 +- .../10k_1company_distribution.csv | 24 +- .../10k_2company_distribution.csv | 15 +- .../10q_1company_distribution.csv | 8 +- .../10q_2company_distribution.csv | 8 +- .../8k_1company_distribution.csv | 14 + .../8k_2company_distribution.csv | 11 + .../sdg/document-grounded-sdg-demo.yaml | 127 ++ .../sdg/document-grounded-sdg-qwen-demo.yaml | 116 + .../sdg/document-grounded-sdg-qwen-prod.yaml | 127 ++ .../workflows/sdg/document-grounded-sdg.yaml | 253 ++- .../workflows/sft/nemotron-3-nano.yaml | 2 +- .../finance/workflows/sft/qwen3_14b.yaml | 2 +- .../finance/workflows/sft/qwen3_4b.yaml | 17 +- .../finance/workflows/sft/qwen3_4b_smoke.yaml | 2 +- .../finance/workflows/sft/sft_presets.yaml | 9 +- nvflow/recipes/multimodal/__init__.py | 18 + .../hopchain_category_identification.txt | 64 + .../prompts/hopchain_image_filter.txt | 51 + .../hopchain_instance_localization_sam3.txt | 1 + .../hopchain_llm_judge_answer_question.txt | 26 + .../prompts/hopchain_query_design.txt | 392 ++++ .../prompts/hopchain_sft_answer_question.txt | 23 + .../prompts/hopchain_sft_trace_judge.txt | 35 + nvflow/recipes/multimodal/recipe.yaml | 13 + nvflow/recipes/multimodal/stages/__init__.py | 26 + .../stages/image_filter/__init__.py | 24 + .../stages/image_filter/image_filter.py | 220 ++ .../recipes/multimodal/stages/sdg/__init__.py | 24 + .../stages/sdg/filter_easy_candidates.py | 118 + .../stages/sdg/generate_multihop_queries.py | 112 + .../stages/sdg/identify_categories.py | 111 + .../stages/sdg/judge_candidate_queries.py | 118 + .../stages/sdg/localize_instances.py | 197 ++ .../sdg/prepare_filtered_image_inputs.py | 107 + .../sdg/preprocess_generation_inputs.py | 239 ++ .../stages/sdg/reconcile_llm_judges.py | 96 + .../sdg/sample_instance_combinations.py | 130 ++ .../stages/sdg/sft_reasoning_traces.py | 348 +++ .../stages/sdg/verify_candidate_queries.py | 94 + .../sdg/visualize_final_hopchain_data.py | 108 + .../tests/test_hopchain_image_filter_paths.py | 218 ++ .../tests/test_hopchain_llm_judge_workflow.py | 295 +++ ...t_hopchain_sample_instance_combinations.py | 518 +++++ .../test_hopchain_sft_reasoning_traces.py | 298 +++ nvflow/recipes/multimodal/utils/__init__.py | 15 + ...ain_category_identification_postprocess.py | 192 ++ ...hain_category_identification_preprocess.py | 91 + .../hopchain_difficulty_filter_postprocess.py | 173 ++ .../hopchain_difficulty_filter_preprocess.py | 115 + .../utils/hopchain_llm_judge_common.py | 117 + .../utils/hopchain_localization_filters.py | 64 + .../utils/hopchain_localize_instances_sam3.py | 659 ++++++ .../hopchain_merge_localization_shards.py | 88 + .../hopchain_query_generation_postprocess.py | 154 ++ .../hopchain_query_generation_preprocess.py | 104 + .../utils/hopchain_reconcile_llm_judges.py | 281 +++ .../utils/hopchain_run_llm_judge.py | 483 ++++ .../hopchain_sample_instance_combinations.py | 726 ++++++ .../multimodal/utils/hopchain_sdg_common.py | 176 ++ .../multimodal/utils/hopchain_sdg_models.py | 296 +++ .../hopchain_sft_trace_filter_postprocess.py | 220 ++ ...pchain_sft_trace_generation_postprocess.py | 157 ++ ...opchain_sft_trace_generation_preprocess.py | 94 + .../hopchain_sft_trace_judge_preprocess.py | 117 + .../hopchain_verify_candidate_queries.py | 139 ++ .../multimodal/utils/image_filter_models.py | 132 ++ .../utils/image_filter_postprocess.py | 236 ++ .../utils/image_filter_preprocess.py | 174 ++ .../recipes/multimodal/utils/image_utils.py | 158 ++ nvflow/recipes/multimodal/utils/inference.py | 133 ++ .../utils/multimodal_model_configs.py | 27 + .../utils/prepare_filtered_image_inputs.py | 170 ++ .../render_hopchain_final_dataset_html.py | 999 +++++++++ .../recipes/multimodal/utils/runtime_env.py | 45 + .../hopchain-image-filter-demo.yaml | 14 + .../image_filter/hopchain-image-filter.yaml | 69 + .../workflows/sdg/hopchain-sdg-demo.yaml | 40 + .../workflows/sdg/hopchain-sdg.yaml | 485 ++++ nvflow/utils/__init__.py | 21 +- nvflow/utils/jsonl.py | 226 ++ pyproject.toml | 65 +- scripts/_dump_rollout_fixtures.py | 153 ++ scripts/_dump_verify_fixtures.py | 109 + scripts/serve_vllm_patched.py | 311 --- scripts/view_traces.py | 615 ++++++ scripts/wandb_consolidate.py | 310 ++- .../rollout/aggregate_cmd_custom_filename.txt | 7 + .../rollout/aggregate_cmd_default.txt | 7 + .../rollout/client_cmd_dual_server.txt | 269 +++ .../rollout/client_cmd_max_samples.txt | 269 +++ .../fixtures/rollout/client_cmd_no_chunk.txt | 269 +++ tests/fixtures/rollout/client_cmd_no_rcp.txt | 267 +++ .../rollout/client_cmd_policy_only.txt | 266 +++ tests/fixtures/rollout/filter_cmd_full.txt | 14 + tests/fixtures/rollout/filter_cmd_minimal.txt | 11 + tests/fixtures/rollout/merge_cmd_1chunk.txt | 78 + tests/fixtures/rollout/merge_cmd_8chunks.txt | 78 + .../verify/analysis_cmd_empty_entries.txt | 6 + .../verify/analysis_cmd_multi_seed.txt | 16 + .../verify/analysis_cmd_single_seed.txt | 11 + .../verify/verify_cmd_local_judge.txt | 94 + .../verify/verify_cmd_openai_judge.txt | 91 + tests/requirements-ci.txt | 18 + tests/test_aggregate_seeds.py | 135 ++ tests/test_apply_validate_filter.py | 196 ++ tests/test_cli_cmd.py | 235 ++ tests/test_dataset_transformer.py | 300 +++ tests/test_enrich_rollouts.py | 255 +++ tests/test_grpo_gym_install.py | 164 ++ tests/test_jsonl.py | 333 +++ tests/test_postprocess_validate.py | 261 +++ tests/test_prompt_template_applier.py | 389 ++++ tests/test_regex_prefilter_sets.py | 241 ++ tests/test_responses_api_converter.py | 353 +++ tests/test_rollout.py | 834 +++++++ tests/test_shuffle_jsonl.py | 243 +++ tests/test_verify.py | 309 +++ uv.lock | 1941 +++++------------ 271 files changed, 34080 insertions(+), 7117 deletions(-) create mode 100644 .dockerignore create mode 100644 dockerfiles/Dockerfile.nemo-gym create mode 100644 dockerfiles/Dockerfile.nvflow delete mode 100644 dockerfiles/Dockerfile.vllm-grpo rename docs/{architecture => }/ARCHITECTURE.md (88%) delete mode 100644 docs/architecture/ARCHITECTURE_INDEX.md delete mode 100644 docs/architecture/ARCHITECTURE_QUICK_REFERENCE.md delete mode 100644 docs/architecture/DIAGRAMS_SUMMARY.md create mode 100644 docs/development/nemo-rl-gym.md create mode 100644 docs/development/sdg/document_grounded/ADDING_A_DOMAIN.md create mode 100644 docs/development/sdg/document_grounded/dgsdg-add-new-domain.png create mode 100644 docs/maintainers/containers.md delete mode 100644 docs/recipes/finance/workflows/06-finance-agent-eval.md create mode 100644 docs/recipes/multimodal/README.md create mode 100644 docs/recipes/multimodal/quick-start.md create mode 100644 docs/remote-launch.md create mode 100644 docs/trace-viewer.md create mode 100644 nvflow/generic_stage/__init__.py create mode 100644 nvflow/generic_stage/sdg/__init__.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/__init__.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/_helpers.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/_schemas.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/_trim_cli.py rename nvflow/{recipes/finance/stages/sdg => generic_stage/sdg/document_grounded}/aggregate_answers.py (66%) rename nvflow/{recipes/finance/stages/sdg => generic_stage/sdg/document_grounded}/dg_sdg_preprocess.py (50%) rename nvflow/{recipes/finance/stages/sdg/document_grounded_data.py => generic_stage/sdg/document_grounded/dgsdg_post_process.py} (57%) create mode 100644 nvflow/generic_stage/sdg/document_grounded/evaluate_answers.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/generate_answers.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/generate_verified_questions.py create mode 100644 nvflow/generic_stage/sdg/document_grounded/gym_genselect_answers.py create mode 100644 nvflow/lib/cli_cmd.py create mode 100644 nvflow/lib/sdg/__init__.py create mode 100644 nvflow/lib/sdg/document_grounded/__init__.py create mode 100644 nvflow/lib/sdg/document_grounded/aggregate.py create mode 100644 nvflow/lib/sdg/document_grounded/analyze_rollouts.py create mode 100644 nvflow/lib/sdg/document_grounded/chunking.py create mode 100644 nvflow/lib/sdg/document_grounded/enrich_rollouts.py create mode 100644 nvflow/lib/sdg/document_grounded/enrich_rollouts_evaluate.py create mode 100644 nvflow/lib/sdg/document_grounded/evaluate.py create mode 100644 nvflow/lib/sdg/document_grounded/genselect.py create mode 100644 nvflow/lib/sdg/document_grounded/gym_configs/sdg_equivalence_judge.yaml create mode 100644 nvflow/lib/sdg/document_grounded/gym_configs/sdg_format_verification.yaml create mode 100644 nvflow/lib/sdg/document_grounded/metadata.py create mode 100644 nvflow/lib/sdg/document_grounded/postprocess.py rename nvflow/{recipes/finance/utils/sdg/document_grounded_preprocess.py => lib/sdg/document_grounded/preprocess.py} (52%) create mode 100644 nvflow/lib/sdg/document_grounded/responses_api.py create mode 100644 nvflow/lib/sdg/document_grounded/sampling.py delete mode 100644 nvflow/lib/vllm_compat.py delete mode 100644 nvflow/recipes/finance/prompts/judge_difficulty.yaml delete mode 100644 nvflow/recipes/finance/stages/sdg/difficulty_estimation.py delete mode 100644 nvflow/recipes/finance/stages/sdg/document_grounded_question_answer_generation_pipeline.py delete mode 100644 nvflow/recipes/finance/stages/sdg/evaluate_answers.py create mode 100644 nvflow/recipes/finance/utils/rl/postprocess_validate.py delete mode 100644 nvflow/recipes/finance/utils/sdg/dgsdg_post_process.py delete mode 100644 nvflow/recipes/finance/utils/sdg/difficulty_estimation.py create mode 100644 nvflow/recipes/finance/utils/sdg/sec_callbacks.py create mode 100644 nvflow/recipes/finance/utils/sdg/sec_postprocess.py create mode 100644 nvflow/recipes/finance/utils/sdg/sec_question_prep.py create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_b1.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_b2.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_b3.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_b4.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_b5.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_smoke1.yaml create mode 100644 nvflow/recipes/finance/workflows/grpo/nemotron_nano_30b_a3b_smoke2.yaml create mode 100644 nvflow/recipes/finance/workflows/sdg/dg_sdg_distribution/8k_1company_distribution.csv create mode 100644 nvflow/recipes/finance/workflows/sdg/dg_sdg_distribution/8k_2company_distribution.csv create mode 100644 nvflow/recipes/finance/workflows/sdg/document-grounded-sdg-demo.yaml create mode 100644 nvflow/recipes/finance/workflows/sdg/document-grounded-sdg-qwen-demo.yaml create mode 100644 nvflow/recipes/finance/workflows/sdg/document-grounded-sdg-qwen-prod.yaml create mode 100644 nvflow/recipes/multimodal/__init__.py create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_category_identification.txt create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_image_filter.txt create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_instance_localization_sam3.txt create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_llm_judge_answer_question.txt create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_query_design.txt create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_sft_answer_question.txt create mode 100644 nvflow/recipes/multimodal/prompts/hopchain_sft_trace_judge.txt create mode 100644 nvflow/recipes/multimodal/recipe.yaml create mode 100644 nvflow/recipes/multimodal/stages/__init__.py create mode 100644 nvflow/recipes/multimodal/stages/image_filter/__init__.py create mode 100644 nvflow/recipes/multimodal/stages/image_filter/image_filter.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/__init__.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/filter_easy_candidates.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/generate_multihop_queries.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/identify_categories.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/judge_candidate_queries.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/localize_instances.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/prepare_filtered_image_inputs.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/preprocess_generation_inputs.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/reconcile_llm_judges.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/sample_instance_combinations.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/sft_reasoning_traces.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/verify_candidate_queries.py create mode 100644 nvflow/recipes/multimodal/stages/sdg/visualize_final_hopchain_data.py create mode 100644 nvflow/recipes/multimodal/tests/test_hopchain_image_filter_paths.py create mode 100644 nvflow/recipes/multimodal/tests/test_hopchain_llm_judge_workflow.py create mode 100644 nvflow/recipes/multimodal/tests/test_hopchain_sample_instance_combinations.py create mode 100644 nvflow/recipes/multimodal/tests/test_hopchain_sft_reasoning_traces.py create mode 100644 nvflow/recipes/multimodal/utils/__init__.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_category_identification_postprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_category_identification_preprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_difficulty_filter_postprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_difficulty_filter_preprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_llm_judge_common.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_localization_filters.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_localize_instances_sam3.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_merge_localization_shards.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_query_generation_postprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_query_generation_preprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_reconcile_llm_judges.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_run_llm_judge.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sample_instance_combinations.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sdg_common.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sdg_models.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sft_trace_filter_postprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sft_trace_generation_postprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sft_trace_generation_preprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_sft_trace_judge_preprocess.py create mode 100644 nvflow/recipes/multimodal/utils/hopchain_verify_candidate_queries.py create mode 100644 nvflow/recipes/multimodal/utils/image_filter_models.py create mode 100644 nvflow/recipes/multimodal/utils/image_filter_postprocess.py create mode 100644 nvflow/recipes/multimodal/utils/image_filter_preprocess.py create mode 100644 nvflow/recipes/multimodal/utils/image_utils.py create mode 100644 nvflow/recipes/multimodal/utils/inference.py create mode 100644 nvflow/recipes/multimodal/utils/multimodal_model_configs.py create mode 100644 nvflow/recipes/multimodal/utils/prepare_filtered_image_inputs.py create mode 100644 nvflow/recipes/multimodal/utils/render_hopchain_final_dataset_html.py create mode 100644 nvflow/recipes/multimodal/utils/runtime_env.py create mode 100644 nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml create mode 100644 nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml create mode 100644 nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml create mode 100644 nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml create mode 100644 nvflow/utils/jsonl.py create mode 100644 scripts/_dump_rollout_fixtures.py create mode 100644 scripts/_dump_verify_fixtures.py delete mode 100644 scripts/serve_vllm_patched.py create mode 100644 scripts/view_traces.py create mode 100644 tests/fixtures/rollout/aggregate_cmd_custom_filename.txt create mode 100644 tests/fixtures/rollout/aggregate_cmd_default.txt create mode 100644 tests/fixtures/rollout/client_cmd_dual_server.txt create mode 100644 tests/fixtures/rollout/client_cmd_max_samples.txt create mode 100644 tests/fixtures/rollout/client_cmd_no_chunk.txt create mode 100644 tests/fixtures/rollout/client_cmd_no_rcp.txt create mode 100644 tests/fixtures/rollout/client_cmd_policy_only.txt create mode 100644 tests/fixtures/rollout/filter_cmd_full.txt create mode 100644 tests/fixtures/rollout/filter_cmd_minimal.txt create mode 100644 tests/fixtures/rollout/merge_cmd_1chunk.txt create mode 100644 tests/fixtures/rollout/merge_cmd_8chunks.txt create mode 100644 tests/fixtures/verify/analysis_cmd_empty_entries.txt create mode 100644 tests/fixtures/verify/analysis_cmd_multi_seed.txt create mode 100644 tests/fixtures/verify/analysis_cmd_single_seed.txt create mode 100644 tests/fixtures/verify/verify_cmd_local_judge.txt create mode 100644 tests/fixtures/verify/verify_cmd_openai_judge.txt create mode 100644 tests/requirements-ci.txt create mode 100644 tests/test_aggregate_seeds.py create mode 100644 tests/test_apply_validate_filter.py create mode 100644 tests/test_cli_cmd.py create mode 100644 tests/test_dataset_transformer.py create mode 100644 tests/test_enrich_rollouts.py create mode 100644 tests/test_grpo_gym_install.py create mode 100644 tests/test_jsonl.py create mode 100644 tests/test_postprocess_validate.py create mode 100644 tests/test_prompt_template_applier.py create mode 100644 tests/test_regex_prefilter_sets.py create mode 100644 tests/test_responses_api_converter.py create mode 100644 tests/test_rollout.py create mode 100644 tests/test_shuffle_jsonl.py create mode 100644 tests/test_verify.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d67219f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,41 @@ +# Build context excludes for dockerfiles/Dockerfile.nvflow (COPY . -> /opt/nvflow). +# The client image bakes the source + a fresh uv venv. We KEEP .git in the context +# (hatch-vcs versioning needs it at build); the image then squashes it to a single +# history-free snapshot commit (nemo-run only needs `git archive HEAD`). We drop +# everything heavy, generated, or secret. Other NVFlow Dockerfiles clone their +# sources (they do not COPY this context), so these excludes are safe for them. + +# Prebuilt/host venv (the image builds its own) and uv cache +.venv/ +**/.venv/ +.uv-cache/ +uv-cache/ + +# Huge untracked local caches / outputs (cache/ is tens of GB) +cache/ +outputs/ +htmlcov/ + +# Python/pytest/tooling artifacts +**/__pycache__/ +*.pyc +*.pyo +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ + +# Editor/local backups. Scope to cluster-config backups only -- a bare *.bak* +# would also drop TRACKED dataset files (e.g. datasets/finance_agent/*.bak), +# which then show up as phantom deletions and mark the baked tree dirty. +cluster_configs/*.bak* +*~ + +# Secrets / personal cluster config (gitignored; must never be baked into an image) +cluster_configs/my_cluster.yaml +**/*.env +.env + +# Container image artifacts that should never enter the context +*.sqsh +*.tar +*.tar.gz diff --git a/.github/workflows/secrets-detector.yml b/.github/workflows/secrets-detector.yml index dcb2ca9..2794a59 100644 --- a/.github/workflows/secrets-detector.yml +++ b/.github/workflows/secrets-detector.yml @@ -30,6 +30,11 @@ jobs: run: | curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin + # --results=verified,unknown fails on confirmed secrets and on anything + # TruffleHog could not check, but drops findings it actively verified as + # not a secret. Without it, any 40-hex string near a keyword trips a + # detector -- pinned upstream git SHAs read as Weights & Biases keys. + # Keep the flags in sync with the secrets-detector job in .gitlab-ci.yml. - name: Scan for secrets run: | - trufflehog git file://. --since-commit ${{ github.event.pull_request.base.sha }} --fail --no-update + trufflehog git file://. --since-commit ${{ github.event.pull_request.base.sha }} --fail --no-update --results=verified,unknown diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d3603af..bb77b63 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -35,9 +35,10 @@ jobs: run: | curl -LsSf https://astral.sh/uv/install.sh | sh uv venv --python 3.12 - uv pip install pytest pytest-cov pytest-timeout - uv pip install PyYAML omegaconf rich + # Dependency list is shared with .gitlab-ci.yml so the two pipelines + # cannot drift. Add test-only deps to tests/requirements-ci.txt. uv pip install -e . --no-deps + uv pip install -r tests/requirements-ci.txt - name: Test run: | diff --git a/.gitignore b/.gitignore index 9d0f4f5..2bc4922 100644 --- a/.gitignore +++ b/.gitignore @@ -25,14 +25,13 @@ wheels/ *.egg # Virtual environments -.venv/ +.venv venv/ ENV/ env/ # UV .uv/ - # IDE .vscode/ .idea/ @@ -46,6 +45,12 @@ env/ htmlcov/ .tox/ +# HF / tokenizer / debug caches. Created by ad-hoc local Python sessions +# (e.g. AutoConfig.from_pretrained(..., trust_remote_code=True)) when +# HF_HOME defaults to $cwd/cache. Pipeline runs use the Lustre HF cache, +# never this directory -- keep it out of git to avoid accidental commits. +/cache/ + # Data and outputs (customize based on your needs) /data /data/ @@ -82,6 +87,7 @@ nvflow/recipes/finance/datasets/finance_agent/*.json # Backup files *.bak +*.bak-* # OS .DS_Store diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 947952c..b5ac76d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,24 @@ dco-check: stage: lint image: python:3.12-slim script: - - python scripts/check_dco.py + - python3 scripts/check_dco.py + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + +# Mirrors .github/workflows/secrets-detector.yml. The GitHub copy only runs on +# pull requests, and this repo opens one per release, so without this job a +# finding stays invisible until release day. Keep the flags in sync with the +# GitHub workflow. +secrets-detector: + stage: lint + image: python:3.12-slim + variables: + GIT_DEPTH: 0 + before_script: + - apt-get update -qq && apt-get install -y -qq git curl > /dev/null + - curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin + script: + - trufflehog git file://. --since-commit "$CI_MERGE_REQUEST_DIFF_BASE_SHA" --fail --no-update --results=verified,unknown rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" @@ -54,11 +71,14 @@ test: stage: test image: python:3.12-slim script: - # Lightweight install: skip heavy core deps (nemo-skills ~200+ packages) - # that unit tests don't need. Only install the project + test deps. + # Lightweight install: the project plus tests/requirements-ci.txt, but not + # the heavy nemo-skills stack (~200 packages incl. torch). Tests that + # genuinely need nemo-skills use pytest.importorskip and skip here; they + # run in the full-deps CI. The dependency list is shared with + # .github/workflows/unit-tests.yml β€” edit it there, in one place. - uv venv --python 3.12 - - uv pip install -e ".[dev]" --no-deps - - uv pip install pytest pytest-cov pytest-xdist pytest-timeout PyYAML omegaconf rich + - uv pip install -e . --no-deps + - uv pip install -r tests/requirements-ci.txt # --no-sync prevents `uv run` from implicitly auto-installing heavy core # deps (nemo-skills etc) that we deliberately skipped above. Without it, # tests would pass here for the wrong reason and fail on GitHub CI. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 483ff73..df3c38c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,5 +43,5 @@ repos: rev: v1.7.1 hooks: - id: mypy - additional_dependencies: [types-PyYAML, types-tqdm] + additional_dependencies: [types-PyYAML, types-tqdm, types-requests] args: [--ignore-missing-imports] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b51dcf6..0d5bde7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,12 +63,12 @@ By making a contribution to this project, I certify that: this project or the open source license(s) involved. ``` -## Merge Requests +## Pull Requests 1. Fork the repository and create a branch from `main`. 2. Make your changes with signed-off commits (`git commit -s`). -3. Push your branch and open a Merge Request into `main`. -4. Ensure the MR description references any related issues and describes the change clearly. +3. Push your branch and open a Pull Request into `main`. +4. Ensure the PR description references any related issues and describes the change clearly. ## Code and Documentation @@ -80,7 +80,6 @@ By making a contribution to this project, I certify that: By contributing, you agree that your contributions will be licensed under the same license as the project: the Apache License, Version 2.0. See [LICENSE](LICENSES/LICENSE) for the full text. -## IP Review and Open Source Compliance +## Third-Party Code -- **Ongoing modifications**: For changes to project code (including contributions by third parties), follow NVIDIA's IP review process: [https://nv/ip_review_process](https://nv/ip_review_process). -- **Open Source compliance**: This project follows NVIDIA OSRB recommendations for Apache 2.0 release. +If your contribution adds or updates a third-party dependency, make sure its license is compatible with Apache 2.0 and record it in [`LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md`](LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md). diff --git a/INSTALL.md b/INSTALL.md index 4b7f59f..2e041a2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,31 +1,54 @@ # Installation & Setup Guide -Quick setup guide for NVFlow - a lightweight orchestration tool for Slurm clusters. NVFlow's containers are self-sufficient β€” all dependencies are pre-installed, so no runtime downloads are needed. Once images and models are staged, the pipeline runs fully offline. +Operator guide for getting NVFlow running on a Slurm cluster: install the client, stage containers and models, configure your cluster, and verify. NVFlow's containers are self-sufficient β€” all dependencies are pre-installed, so once images and models are staged, the pipeline runs fully offline. -## πŸ“‹ Steps +> This guide sets up the **cluster side**. How you run the `nflow` **client** β€” +> local install or the airgapped `nvflow-client` container, and whether it +> submits directly or over an SSH tunnel β€” is summarized in +> [Choose your client setup](#choose-your-client-setup) just below. -1. [Prerequisites](#prerequisites) -2. [Setup Containers](#setup-containers) -3. [Download Models](#download-models) -4. [Setup NeMo-RL & NeMo-Gym Sources (for GRPO)](#setup-nemo-rl--nemo-gym-sources-for-grpo) -5. [Configure Your Cluster](#configure-your-cluster) -6. [Verify Installation](#verify-installation) +> **Building artifacts?** Producing the container images is a maintainer task and +> lives under [`docs/maintainers/`](docs/maintainers/), not on this page. ---- +## Choose your client setup -## Prerequisites +`nflow` only submits Slurm jobs β€” the heavy work runs on the cluster. Two +independent choices decide how you run it: **how you provision the client**, and +**how it reaches Slurm**. + +| You run `nflow` on… | Provision the client | Reach Slurm | Guide | +|---|---|---|---| +| Cluster login/dev node (internet) | `uv sync` | direct | [README](README.md#-installation) | +| Laptop / dev box (internet) | `uv sync` | SSH tunnel | [remote-launch](docs/remote-launch.md) | +| Anywhere, airgapped / no install | `nvflow-client` container | direct or SSH tunnel | [remote-launch](docs/remote-launch.md) | + +Steps 1–6 below are the **cluster side** (stage images/models, write +`my_cluster.yaml`, verify) and apply to every row above. -> **Note:** This guide assumes you've already completed the [README.md](README.md) setup (installed `uv`, cloned the repo, ran `uv sync`). +## πŸ“‹ What you'll do -### Build Host Requirements (for building container images) +Six steps, top to bottom. Each step below opens with a **Goal** and ends with a **βœ… Done when** check so you always know where you are. -The `docker build` step needs **internet access** to pull base layers, source from GitHub, and packages from PyPI / NGC / Docker Hub. The resulting `.sqsh` files then run fully offline on the cluster. +| Step | What it does | Who needs it | +|------|--------------|--------------| +| 1. [Prerequisites](#prerequisites) | Confirm cluster access + required tools | Everyone | +| 2. [Setup Containers](#setup-containers) | Stage the five `.sqsh` images on the cluster | Everyone | +| 3. [Download Models](#download-models) | Pre-stage the HF models your workflows use | Everyone | +| 4. [GRPO Prerequisites](#grpo-prerequisites) | SEC cache prefetch | **GRPO only β€” else skip** | +| 5. [Configure Your Cluster](#configure-your-cluster) | Write `cluster_configs/my_cluster.yaml` | Everyone | +| 6. [Verify Installation](#verify-installation) | Sanity-check the whole setup | Everyone | -- **Docker Engine** or **Docker Desktop** (any OS - Linux, macOS, Windows/WSL2) -- **`docker login nvcr.io`** - required once for the NeMo-RL base image -- **`docker buildx`** - only needed for multi-arch / cross-arch builds (ships with Docker Desktop; on Linux: `docker buildx version`) +> **Shortcut:** if a maintainer already staged the `.sqsh` images and models for you, you only need Steps 1, 5, and 6. + +--- + +## Prerequisites -> **Note:** If your destination cluster is `linux/amd64` (the common case) and your build host is amd64 Linux / Intel macOS / Windows, the default `docker build` works without `buildx`. +**Step 1 of 6 Β· Goal:** confirm you can reach the cluster and have the tools the setup needs. + +> **Note:** This guide assumes you've already installed the client (see the [README](README.md#-installation): install `uv`, clone the repo, run `uv sync`). +> +> πŸ”Œ **Airgapped / no internet on the install host?** Skip the local install and drive `nflow` from the prebuilt `nvflow-client` container (CLI + venv baked in, no `uv sync`, no client internet) β€” see **[docs/remote-launch.md](docs/remote-launch.md)**. You still stage the worker images and models on the cluster (Steps 2–3 below); only the install differs. ### Cluster Setup Requirements @@ -39,7 +62,9 @@ The `docker build` step needs **internet access** to pull base layers, source fr > **Note:** If you already have `.sqsh` container images staged on the cluster, skip to [Configure Your Cluster](#configure-your-cluster). -**yq (YAML parser):** +
+Install yq (only needed for the parallel conversion script) + ```bash # Check if installed yq --version @@ -59,6 +84,7 @@ chmod +x $HOME/bin/yq echo 'export PATH="$HOME/bin:$PATH"' >> $HOME/.bashrc source $HOME/.bashrc ``` +
**curl & enroot:** ```bash @@ -72,183 +98,31 @@ enroot version # Run on cluster node - Slurm version: `scontrol show config | grep SLURM_VERSION` (25.x needs the enroot Ray template fix - see [Troubleshooting](#troubleshooting)) - Storage paths for data/models/containers +**βœ… Done when:** `enroot version` works on a cluster node and you know your Slurm account, a partition, and your storage paths. + --- ## Setup Containers -NVFlow uses five containers converted to `.sqsh` format for running on Slurm clusters. **Four are built locally** from self-contained Dockerfiles in [`dockerfiles/`](dockerfiles/); the fifth (`sglang`) is pulled as-is. - -**Required containers (5):** - -| Container | Source | Tested Version | Action | -|-----------|--------|----------------|--------| -| `nvflow-nemo-rl` | [`dockerfiles/Dockerfile.nemo-rl`](dockerfiles/Dockerfile.nemo-rl) | base `nvcr.io/nvidia/nemo-rl:v0.6.0` | **Build** (see Step 1) | -| `nvflow-nemo-skills` | [`dockerfiles/Dockerfile.nemo-skills`](dockerfiles/Dockerfile.nemo-skills) | NeMo-Skills @ `0229040` | **Build** (see Step 1) | -| `nvflow-vllm` | [`dockerfiles/Dockerfile.vllm`](dockerfiles/Dockerfile.vllm) | base `vllm/vllm-openai:v0.18.1` | **Build** (SDG/eval) | -| `nvflow-vllm-grpo` | [`dockerfiles/Dockerfile.vllm-grpo`](dockerfiles/Dockerfile.vllm-grpo) | base `vllm/vllm-openai:v0.17.1` | **Build** (GRPO rollouts/judge) | -| `sglang` | Docker Hub | `lmsysorg/sglang:v0.5.10.post1` | **Pull** (no custom Dockerfile) | - -> **Note:** All four custom images are **built**, not pulled. The four Dockerfiles bake in NeMo-Skills source, NeMo-Gym source, pre-built virtual environments, `tiktoken` / `openai_harmony` encoding caches, and `/root/.local β†’ /opt/uv-python` path relocation so the images run cleanly under `enroot`/`pyxis` on Slurm with no outbound network access. - -**Optional containers** (not currently used by any NVFlow recipes): - -| Container | Source | Action | -|-----------|--------|--------| -| `megatron` | NeMo-Skills Dockerfiles | Build | -| `sandbox` | NeMo-Skills Dockerfiles | Build | -| `verl` | NeMo-Skills Dockerfiles | Build | -| `trtllm` | `nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc8` | Pull from NGC | - -### Step 1: Build Docker Images - -NVFlow ships self-contained Dockerfiles in [`dockerfiles/`](dockerfiles/) that pre-install all Python packages, pre-cache tokenizer encodings, and pre-build virtual environments. Run `docker build` on a connected build host: - -```bash -cd /path/to/nvflow - -# Build all four custom images (amd64, the common case) -docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:v0.6.0 . -docker build -f dockerfiles/Dockerfile.nemo-skills -t nvflow-nemo-skills:0229040 . -docker build -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:v0.18.1 . -docker build -f dockerfiles/Dockerfile.vllm-grpo -t nvflow-vllm-grpo:v0.17.1 . - -# sglang is pulled as-is, no custom Dockerfile -docker pull lmsysorg/sglang:v0.5.10.post1 -``` - -> **Tip:** The Dockerfiles expose `ARG`s for version pins (`NEMO_SKILLS_COMMIT`, `NEMO_GYM_BRANCH`, `VLLM_VERSION`, `BASE_IMAGE`). Defaults are listed in [`dockerfiles/README.md`](dockerfiles/README.md#version-pins). Keep `NEMO_SKILLS_COMMIT` consistent across `Dockerfile.nemo-skills`, `Dockerfile.nemo-rl`, and `pyproject.toml`. - -For **cross-arch builds** (e.g. building an `amd64` image on Apple Silicon, or a multi-arch manifest list pushed directly to a registry), see [`dockerfiles/docker_instructions.md`](dockerfiles/docker_instructions.md#1-build). Cross-arch builds use `docker buildx` with QEMU emulation and are significantly slower than native. - -For optional containers (`megatron`, `sandbox`, `verl`), build them from the upstream [NeMo-Skills Dockerfiles](https://github.com/NVIDIA-NeMo/Skills/tree/022904023ad7a83a87662a313cf72e7df5891d55/dockerfiles). - -### Step 1b: Sanity-Check Images Before Conversion - -Before the time-consuming `enroot import` step, run the smoke checks in [`dockerfiles/docker_instructions.md` Β§2](dockerfiles/docker_instructions.md#2-sanity-checks-blockers). Each check is a **hard blocker** - if it fails locally, the image will not work in production. They verify the offline-critical pieces: `uv` works offline, the 6 Gym `.venv` symlinks are intact, `tiktoken` / `openai_harmony` caches load with `--network=none`, and `tzdata` is populated. - -### Step 2: Push Images to a Registry (Option A) or Save as Tarball (Option B) - -`enroot` runs on Slurm compute/login nodes (Linux only). There are two paths from a Docker image to a `.sqsh` file - pick whichever fits your offline workflow. - -#### Option A: Via a private container registry (recommended) - -Push the built images to a registry accessible from your cluster (Docker Hub, NGC, or a private registry): - -```bash -REGISTRY= - -docker tag nvflow-nemo-rl:v0.6.0 $REGISTRY/nvflow-nemo-rl:v0.6.0 -docker tag nvflow-nemo-skills:0229040 $REGISTRY/nvflow-nemo-skills:0229040 -docker tag nvflow-vllm:v0.18.1 $REGISTRY/nvflow-vllm:v0.18.1 -docker tag nvflow-vllm-grpo:v0.17.1 $REGISTRY/nvflow-vllm-grpo:v0.17.1 - -docker push $REGISTRY/nvflow-nemo-rl:v0.6.0 -docker push $REGISTRY/nvflow-nemo-skills:0229040 -docker push $REGISTRY/nvflow-vllm:v0.18.1 -docker push $REGISTRY/nvflow-vllm-grpo:v0.17.1 - -# sglang can be pulled directly by enroot (no push needed unless your -# cluster cannot reach Docker Hub). -``` - -> **Why push?** Slurm cluster nodes typically don't have Docker installed, so `enroot` needs to pull images from a registry (or load them from a Docker daemon - see Option B). - -#### Option B: Via a saved tarball (no registry required) - -For offline sites without a private registry, save the Docker image to a tarball, transfer it to a Linux host that has both Docker and `enroot`, load the tarball into the local Docker daemon, then import via `dockerd://`: - -```bash -# On the build host -docker save nvflow-nemo-rl:v0.6.0 | gzip > nvflow-nemo-rl-v0.6.0.tar.gz -docker save nvflow-nemo-skills:0229040 | gzip > nvflow-nemo-skills-0229040.tar.gz -docker save nvflow-vllm:v0.18.1 | gzip > nvflow-vllm-v0.18.1.tar.gz -docker save nvflow-vllm-grpo:v0.17.1 | gzip > nvflow-vllm-grpo-v0.17.1.tar.gz - -# Transfer the .tar.gz files to the cluster (scp / rsync / sneakernet) -``` - -`enroot import` natively supports only `docker://` (remote registry), `dockerd://` (local Docker daemon), and `podman://` URIs. If the cluster has neither a private registry nor a Docker daemon, run a transient local registry container, push to it, and import via `docker://localhost:5000/...`. - -### Step 3: Update Container Config - -Copy the template to a personal file that records the registry / tag references the cluster should pull from: - -```bash -cp cluster_configs/containers.yaml cluster_configs/my_containers.yaml -``` - -Edit `cluster_configs/my_containers.yaml` with your registry paths. The YAML **keys** (`nemo-skills`, `nemo-rl`, `vllm`, `vllm-grpo`, `sglang`) match what the workflow code references and must not be renamed; only the registry / tag values change: - -```yaml -containers: - nemo-rl: your-registry/nvflow-nemo-rl:v0.6.0 - nemo-skills: your-registry/nvflow-nemo-skills:0229040 - vllm: your-registry/nvflow-vllm:v0.18.1 # v0.18.1 for SDG/eval - vllm-grpo: your-registry/nvflow-vllm-grpo:v0.17.1 # v0.17.1 for GRPO rollouts/judge - sglang: lmsysorg/sglang:v0.5.10.post1 -``` - -> **Note:** `my_containers.yaml` is gitignored (`cluster_configs/*.yaml` pattern), so your registry paths stay local and won't be committed. - -### Step 4: Convert to .sqsh Format - -#### Option A: Automated Setup (Recommended, for Option A registries) +**Step 2 of 6 Β· Goal:** have the five `.sqsh` container images staged on your cluster, with their paths in hand. -Use the setup script to download from your registry and convert all containers in parallel. Pass your personal config with `--config`: +NVFlow runs its cluster jobs inside five `.sqsh` container images. As an operator you only need the `.sqsh` files **staged on your cluster** and their paths recorded in your cluster config. -```bash -# Run from a cluster login node (sbatch requires Slurm access) -sbatch --account=YOUR_ACCOUNT scripts/setup_containers.sh --config cluster_configs/my_containers.yaml ./containers -``` - -The `--config` flag is required - the script reads image references from the specified YAML file, pulls them via `enroot`, and converts to `.sqsh` format. See [the script](scripts/setup_containers.sh) for additional options (`--platform`, `--force`). - -**Check progress:** -```bash -tail -f outputs/logs/slurm-containers-.out -``` +- **Already have `.sqsh` files staged** (by a maintainer or a previous setup)? Note their paths and skip to [Configure Your Cluster](#configure-your-cluster). +- **Need to build / convert them yourself?** See **[docs/maintainers/containers.md](docs/maintainers/containers.md)** β€” build host requirements, `docker build`, push/save, and `enroot import` to `.sqsh`. -#### Option B: Manual Conversion +Your cluster config references the images by fixed keys β€” `nemo-rl`, `nemo-skills`, `vllm`, `vllm-grpo`, and `sglang`. The build guide's "Update Container Config" step explains how to set them; [Configure Your Cluster](#configure-your-cluster) ties them into your run config. -Convert images one at a time using `enroot` on a cluster node. From a registry, use `docker://$REGISTRY/...`; from a loaded tarball, use `dockerd://...` after `docker load`: - -```bash -CONTAINER_DIR= - -enroot import --output $CONTAINER_DIR/nvflow-nemo-rl-v0.6.0.sqsh \ - "docker://$REGISTRY/nvflow-nemo-rl:v0.6.0" # from a registry -# -- or -- -gunzip -c nvflow-nemo-rl-v0.6.0.tar.gz | docker load -enroot import --output $CONTAINER_DIR/nvflow-nemo-rl-v0.6.0.sqsh \ - dockerd://nvflow-nemo-rl:v0.6.0 # from a tarball -``` - -Repeat for `nemo-skills`, `vllm`, `vllm-grpo`, and (if needed) `sglang`. - -**Two things to watch for:** - -- **Registries with a path component need `#` instead of `/`.** `enroot` parses `docker:///` such that everything after the first `/` is image path, which breaks for registries where the host itself contains a path (e.g. `nvcr.io/`). Use `#` to separate host from image path: - ```bash - enroot import --output nvflow-vllm-v0.18.1.sqsh \ - "docker://nvcr.io#/nvflow-vllm:v0.18.1" - ``` -- **Filename colon.** `enroot` writes the Docker tag separator (`:`) literally into the output filename. Either pass `--output` with a shell-safe name (as above) or rename after import: - ```bash - mv "nvflow-nemo-rl:v0.6.0.sqsh" nvflow-nemo-rl-v0.6.0.sqsh - ``` - -If the cluster authenticates to your registry, drop credentials into `~/.config/enroot/.credentials`: - -``` -machine login password -``` +> **`nemo-gym` (needed for GRPO & DG-SDG):** the Gym-only stages β€” GRPO `prepare_data` / `prefetch_cache` and the DG-SDG gym stages β€” run in a dedicated **CPU-only** `nemo-gym` image ([`dockerfiles/Dockerfile.nemo-gym`](docs/maintainers/containers.md#gym-worker-cpu-only)). Stage this sixth image if you run **GRPO or DG-SDG**; **SFT-only** and **eval-only** runs don't need it. -Move the resulting `.sqsh` files to your cluster's container storage path. +**βœ… Done when:** five `.sqsh` files exist on the cluster and you have their absolute paths for the config. --- ## Download Models +**Step 3 of 6 Β· Goal:** pre-download the models your chosen workflows need to the cluster's HF models directory. + > ⚠️ **Important:** Pre-download models to your cluster storage before running workflows. The runtime sets `HF_HUB_OFFLINE=1`, so any model not already on disk will fail at job time. > > **Why this matters:** @@ -314,7 +188,8 @@ stage_kwargs: server_type: sglang ``` -**Models needed per workflow:** +
+Which models does each workflow need? | Model | Demo SDG | Demo SFT | Demo GRPO | Demo Eval | Production GRPO | |-------|:--------:|:--------:|:---------:|:---------:|:---------------:| @@ -325,6 +200,7 @@ stage_kwargs: | `Qwen/Qwen3-30B-A3B` | | | | | βœ“ | **Tip:** Download commonly used models once and reuse across all workflows. +
### One-Time Connected-Node Stages (Datasets) @@ -337,46 +213,21 @@ A handful of stages legitimately need internet on **first** run to pull benchmar | `workflow-1 step-0 prepare_data` (eval) | HF `secque`, `financebench` | Benchmark data | | `workflow-5 step-4 prepare_data` (GRPO) | HF | Only if `should_download: true` | -For these stages, **temporarily clear** the three HF offline flags (`HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE`, `TRANSFORMERS_OFFLINE`) in your cluster config. Keep `UV_OFFLINE=true` set - `uv` should never need to resolve packages at runtime. +For these stages, **temporarily clear** the three HF offline flags (`HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE`, `TRANSFORMERS_OFFLINE`) in your cluster config. `UV_OFFLINE` is unrelated β€” leave it at its default (unset); none of these stages invoke `uv`. > **Note:** `huggingface_hub` interprets `TRANSFORMERS_OFFLINE=1` as `HF_HUB_OFFLINE=1`, so all three need to be off (or unset) for HF dataset pulls to succeed. ---- - -## Setup NeMo-RL & NeMo-Gym Sources (for GRPO) - -> **Skip this section** if you're using the self-sufficient `nvflow-nemo-rl` image as-is (the recommended path). The image already contains the NeMo-RL source, a pinned NeMo-Gym branch, and a pre-built `.venv` symlinked across all 6 Gym components. No host clones or overlay mounts are required for SDG, SFT, GRPO, or eval workflows. - -This section is **dev mode only** - read it only if you're actively iterating on NeMo-RL or NeMo-Gym source against the self-sufficient image. - -### What the self-sufficient image already contains - -`nvflow-nemo-rl` is built from [`dockerfiles/Dockerfile.nemo-rl`](dockerfiles/Dockerfile.nemo-rl) on top of `nvcr.io/nvidia/nemo-rl:v0.6.0` and bakes in: +**βœ… Done when:** the models for your workflow are on disk under your mounted `hf_models` directory. -- NeMo-Skills @ `0229040` installed into the frozen `/opt/nemo_rl_venv` -- NeMo-Gym source at `/opt/NeMo-RL/3rdparty/Gym-workspace/Gym`, checked out at the `ude/finance-sec-search-v2` branch (override via `NEMO_GYM_BRANCH` build arg) -- A pre-built Gym `.venv` symlinked across all 6 components (`equivalence_llm_judge`, `finance_sec_search`, `simple_agent`, `finance_agent`, `openai_model`, `vllm_model`) -- `/root/.local/share/uv/python` relocated to `/opt/uv-python` and `/root/.local/bin` to `/opt/uv-bin` so the venvs survive enroot/pyxis mounting `$HOME` over `/root` -- `tiktoken` / `openai_harmony` encoding caches at `/opt/tiktoken_cache` - -GRPO stages call `installation_command: source /opt/NeMo-RL/3rdparty/Gym-workspace/Gym/.venv/bin/activate` and find everything they need inside the image. - -### Do NOT overlay-mount source over the image paths - -Bind-mounting a host clone at `/opt/NeMo-RL` or `/opt/NeMo-RL/3rdparty/Gym-workspace/Gym` **shadows the baked `.venv`**, and `installation_command` fails with `No such file or directory` - breaking `prepare_data`, `collect_rollouts`, `compute_rewards`, and `training` for GRPO. +--- -The older dev-mode overlay snippets in `template-slurm.yaml` are commented out for exactly this reason: +## GRPO Prerequisites -```yaml -mounts: - # DO NOT use these with the self-sufficient image β€” they shadow the baked .venv - # - :/opt/NeMo-RL - # - :/opt/NeMo-RL/3rdparty/Gym-workspace/Gym -``` +**Step 4 of 6 Β· GRPO only β€” skip this entire step if you're not running GRPO.** -### Dev mode: iterating on NeMo-RL / NeMo-Gym source +**There are no sources to clone.** The `nvflow-nemo-rl` trainer image bakes NeMo-RL together with one prebuilt NeMo-Gym venv per component, so GRPO `training` resolves no packages at job runtime and needs no bind-mount. The Gym-only stages (`collect_rollouts`, `compute_rewards`, `prefetch_cache`, `prepare_data`) run on the equally self-contained `nvflow-nemo-gym` image. -If you really need to iterate on NeMo-RL or NeMo-Gym source against this image, clone the source trees (NeMo-RL at `v0.6.0` with submodules, NeMo-Gym at `ude/finance-sec-search-v2`), uncomment the two overlay mounts in `cluster_configs/my_cluster.yaml`, and set `NRL_FORCE_REBUILD_VENVS=true` in `env_vars` so Ray workers rebuild their venvs against your source. Your host clone must contain a `.venv` ABI-compatible with the image, and `NRL_FORCE_REBUILD_VENVS=true` **requires internet** at job time β€” only use it on a connected node, never in production. +> How that image is built, and its internals, are covered in **[docs/development/nemo-rl-gym.md](docs/development/nemo-rl-gym.md)**. ### Prefetch SEC Filings Cache (for `finance_sec_search`) @@ -384,28 +235,32 @@ If using the `finance_sec_search` NeMo-Gym environment, you must prefetch the SE The GRPO workflow includes a dedicated `prefetch_cache` stage that runs on a connected node and populates the cache under your `workflow-5-grpo/` output directory. See [`docs/recipes/finance/workflows/06-grpo.md`](docs/recipes/finance/workflows/06-grpo.md) for the full prefetch flow. +**βœ… Done when:** (GRPO users) the SEC filings cache is prefetched if you use `finance_sec_search`. Everyone else: nothing to do β€” move on. + --- ## Configure Your Cluster -### Step 1: Create Your Cluster Config +**Step 5 of 6 Β· Goal:** create and fill in `cluster_configs/my_cluster.yaml`. + +### Create Your Cluster Config ```bash # Copy template cp cluster_configs/template-slurm.yaml cluster_configs/my_cluster.yaml ``` -### Step 2: Edit Your Config +### Edit Your Config Edit `cluster_configs/my_cluster.yaml` and replace all `` values: 1. **SSH settings** - Your cluster login node, username, SSH key path (ONLY for remote job submission from local machine) 2. **Slurm account/partition** - Run `sacctmgr show associations user=$USER` and `sinfo` 3. **Container paths** - Copy from `outputs/logs/slurm-containers-.out` after running setup_containers.sh -4. **Mount points** - Map your cluster paths to container paths (at minimum `:/hf_models` and `:/workspace`) +4. **Mount points** - Map your cluster paths to container paths (at minimum `:/hf_models` and a writable data dir `:/workspace` for outputs + caches). Recipe code and assets ship via the nemo-run packaged snapshot (`/nemo_run/code`), so the repo is not mounted β€” see [Mount Points](docs/cluster-configuration.md#mounts). 5. **Environment variables** - Set `HF_HOME` to a path visible inside the container (see [env_vars docs](docs/cluster-configuration.md#environment-variables)) and any API keys -### Step 3: Keep the Air-Gap Enforcement Block +### Keep the Air-Gap Enforcement Block `template-slurm.yaml` ships with the offline flags pre-populated - leave them on: @@ -415,16 +270,15 @@ env_vars: - HF_HUB_OFFLINE=1 - HF_DATASETS_OFFLINE=1 - TRANSFORMERS_OFFLINE=1 - - UV_OFFLINE=true + # UV_OFFLINE: left UNSET (global flag). The images bake every venv they need, + # so no stage resolves packages at runtime either way. + # - UV_OFFLINE=true # Pre-baked tiktoken / openai_harmony cache (set as ENV in vllm/vllm-grpo # already; setting here applies them uniformly to nemo-skills and nemo-rl) - TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache - TIKTOKEN_RS_CACHE_DIR=/opt/tiktoken_cache - TIKTOKEN_ENCODINGS_BASE=/opt/tiktoken_cache - - # Do NOT set in self-sufficient mode β€” forces Ray workers to re-resolve via uv - # - NRL_FORCE_REBUILD_VENVS=true ``` For the one-time connected-node stages listed in [Download Models](#one-time-connected-node-stages-datasets) above, comment out the three `HF_*_OFFLINE` flags just for that submission, then re-enable. @@ -433,10 +287,14 @@ For the one-time connected-node stages listed in [Download Models](#one-time-con > > πŸ“– **For detailed documentation of all configuration fields, see the [Cluster Configuration Guide](docs/cluster-configuration.md)**. +**βœ… Done when:** `my_cluster.yaml` has no remaining `` values and keeps the air-gap enforcement block. + --- ## Verify Installation +**Step 6 of 6 Β· Goal:** confirm the whole setup before running a real workflow. + ```bash # 1. Test NeMo-Skills import uv run python -c "from nemo_skills.pipeline.cli import generate; print('βœ… OK')" @@ -454,6 +312,8 @@ uv run python -c "from omegaconf import OmegaConf; OmegaConf.load('cluster_confi uv run nflow list-stages ``` +**βœ… Done when:** all five checks above pass. You're ready to run a workflow β€” see [Next Steps](#next-steps). + --- ## Troubleshooting @@ -471,20 +331,7 @@ uv sync --reinstall ``` ### yq not found -```bash -# macOS -brew install yq - -# Linux (auto-detects architecture) -mkdir -p $HOME/bin -ARCH=$(uname -m); case "$ARCH" in x86_64) ARCH=amd64 ;; aarch64) ARCH=arm64 ;; armv7l) ARCH=arm ;; i686) ARCH=386 ;; esac -wget "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${ARCH}" -O $HOME/bin/yq -chmod +x $HOME/bin/yq - -# Add to PATH if needed -echo 'export PATH="$HOME/bin:$PATH"' >> $HOME/.bashrc -source $HOME/.bashrc -``` +`yq` is only needed for the parallel `.sqsh` conversion script. Install it per the [Prerequisites β†’ Install yq](#prerequisites) block above. ### enroot not available ```bash @@ -493,11 +340,8 @@ module load enroot # if available # Or contact your cluster admin ``` -### Docker build fails on `docker login` for NeMo-RL base image -The `nvflow-nemo-rl` build pulls from `nvcr.io/nvidia/nemo-rl:v0.6.0` (NGC). Run `docker login nvcr.io` once (username `$oauthtoken`, password = your [NGC API key](https://ngc.nvidia.com/setup/api-key)). - -### `enroot import` quirks (filename colon, `#` separator for `nvcr.io`) -See [Two things to watch for](#step-4-convert-to-sqsh-format) in Step 4. +### Container build / conversion issues +Building images, `docker login nvcr.io`, and `enroot import` quirks (filename colon, `#` separator for `nvcr.io`) are covered in **[docs/maintainers/containers.md](docs/maintainers/containers.md)**. ### SSH connection failed ```bash @@ -521,29 +365,7 @@ For symptoms specific to the self-sufficient runtime - GRPO `installation_comman ### Ray Cluster Initialization Hangs -**Problem:** Ray cluster hangs during initialization, workers fail to connect - -**Symptoms:** -- Training jobs hang after "Starting Ray cluster" -- Error: `execve(): bad interpreter: No such file or directory` -- Ray workers show connection failures in logs - -**Cause:** SLURM container reattachment issues (SLURM 25.x may be affected) - -**Solution:** -Add `ray_template` to your cluster config: - -```yaml -# In cluster_configs/my_cluster.yaml -executor: slurm -ray_template: "ray_enroot.sub.j2" # Fixes Ray cluster initialization -``` - -**Check your SLURM version:** -```bash -scontrol show config | grep SLURM_VERSION -# Confirmed: SLURM 25.11.2 needs this fix, SLURM 24.x works without it -``` +If training jobs hang after "Starting Ray cluster" (or you see `execve(): bad interpreter: No such file or directory`), your Slurm likely needs the enroot Ray template: set `ray_template: "ray_enroot.sub.j2"` in `my_cluster.yaml`. Full symptoms, cause, and SLURM-version notes are in [cluster-configuration.md β†’ Ray Cluster Configuration](docs/cluster-configuration.md#ray-cluster-configuration). --- @@ -563,11 +385,12 @@ Then head back to the [README.md](README.md#-quick-start) Quick Start section to ## Reference +- **Build the containers**: [`docs/maintainers/containers.md`](docs/maintainers/containers.md) +- **NeMo-RL / NeMo-Gym trainer image & Gym venvs**: [`docs/development/nemo-rl-gym.md`](docs/development/nemo-rl-gym.md) - **NVFlow Dockerfiles**: [`dockerfiles/README.md`](dockerfiles/README.md) - **NVFlow Self-Sufficient Build / Deploy Guide**: [`dockerfiles/docker_instructions.md`](dockerfiles/docker_instructions.md) - **Cluster Configuration Guide**: [`docs/cluster-configuration.md`](docs/cluster-configuration.md) - **NeMo-Skills**: https://github.com/NVIDIA-NeMo/Skills -- **NeMo-Skills Dockerfiles (upstream reference)**: https://github.com/NVIDIA-NeMo/Skills/tree/022904023ad7a83a87662a313cf72e7df5891d55/dockerfiles - **NeMo-RL**: https://github.com/NVIDIA-NeMo/RL - **NeMo-Gym**: https://github.com/NVIDIA-NeMo/Gym - **Official Container Config (NeMo-Skills)**: https://github.com/NVIDIA-NeMo/Skills/blob/main/cluster_configs/example-slurm.yaml diff --git a/LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md b/LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md index d3a6be2..8e13e68 100644 --- a/LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md +++ b/LICENSES/THIRD_PARTY_SW_LICENSE_INFO.md @@ -13,22 +13,22 @@ This document lists third-party open source and other software packages used in | wrapt | 2.1.0 | BSD | [License](https://raw.githubusercontent.com/GrahamDumpleton/wrapt/develop/LICENSE) | | websockets | 15.0.1 | BSD | [License](https://github.com/python-websockets/websockets/blob/main/LICENSE) | | wcwidth | 0.5.3 | MIT | [License](https://raw.githubusercontent.com/jquast/wcwidth/master/LICENSE) | -| wandb | 0.24.1 | MIT | [License](https://github.com/wandb/wandb/blob/main/LICENSE) | +| wandb | 0.28.1 | MIT | [License](https://github.com/wandb/wandb/blob/main/LICENSE) | | virtualenv | 20.36.1 | MIT | [License](https://raw.githubusercontent.com/pypa/virtualenv/main/LICENSE) | -| urllib3 | 1.26.20 | MIT | [License](https://github.com/urllib3/urllib3/blob/main/LICENSE.txt) | +| urllib3 | 2.7.0 | MIT | [License](https://github.com/urllib3/urllib3/blob/main/LICENSE.txt) | | typing-inspection | 0.4.2 | MIT | [License](https://github.com/pydantic/typing-inspection/blob/main/LICENSE) | | typing-inspect | 0.9.0 | MIT | [License](https://raw.githubusercontent.com/ilevkivskyi/typing_inspect/refs/heads/master/LICENSE) | | types-PyYAML | 6.0.12.20250915 | Apache 2.0 | [License](https://pypi.org/project/types-PyYAML/) | | typer-slim | 0.21.1 | MIT | [License](https://raw.githubusercontent.com/fastapi/typer/master/LICENSE) | -| typer | 0.19.2 | MIT | [License](https://raw.githubusercontent.com/fastapi/typer/master/LICENSE) | -| triton | 3.6.0 | MIT | [License](https://github.com/triton-lang/triton/blob/main/LICENSE) | +| typer | 0.21.1 | MIT | [License](https://raw.githubusercontent.com/fastapi/typer/master/LICENSE) | +| triton | 3.7.1 | MIT | [License](https://github.com/triton-lang/triton/blob/main/LICENSE) | | tree-sitter-yaml | 0.7.2 | MIT | [License](https://github.com/ikatyang/tree-sitter-yaml/blob/master/LICENSE) | | tree-sitter-language-pack | 0.13.0 | Apache 2.0 | [License](https://github.com/Goldziher/tree-sitter-language-pack/blob/main/LICENSE) | | tree-sitter-language-pack | 0.13.0 | MIT | [License](https://github.com/Goldziher/tree-sitter-language-pack/blob/main/LICENSE) | | tree-sitter-embedded-template | 0.25.0 | MIT | [License](https://raw.githubusercontent.com/tree-sitter/tree-sitter-embedded-template/master/LICENSE) | | tree-sitter-c-sharp | 0.23.1 | MIT | [License](https://github.com/tree-sitter/tree-sitter-c-sharp/blob/master/LICENSE) | | tree-sitter | 0.25.2 | MIT | [License](https://raw.githubusercontent.com/tree-sitter/py-tree-sitter/master/LICENSE) | -| transformers | 4.57.1 | Apache 2.0 | [License](https://github.com/huggingface/transformers/blob/main/LICENSE) | +| transformers | 5.13.1 | Apache 2.0 | [License](https://github.com/huggingface/transformers/blob/main/LICENSE) | | tqdm | 4.67.2 | MIT and MPL | [License](https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE) | | torchx | 0.7.0 | BSD | [License](https://github.com/meta-pytorch/torchx/blob/main/LICENSE) | | tomlkit | 0.13.3 | MIT | [License](https://github.com/sdispater/tomlkit/blob/master/LICENSE) | @@ -39,7 +39,7 @@ This document lists third-party open source and other software packages used in | tabulate | 0.9.0 | MIT | [License](https://github.com/astanin/python-tabulate/blob/master/LICENSE) | | stack-data | 0.6.3 | MIT | [License](https://pypi.org/project/stack-data/) | | sse-starlette | 3.2.0 | BSD | [License](https://github.com/sysid/sse-starlette/blob/main/LICENSE) | -| soupsieve | 2.8.3 | MIT | [License](https://github.com/facelessuser/soupsieve/blob/main/LICENSE.md) | +| soupsieve | 2.9.1 | MIT | [License](https://github.com/facelessuser/soupsieve/blob/main/LICENSE.md) | | sniffio | 1.3.1 | Apache 2.0 | [License](https://github.com/python-trio/sniffio/blob/master/LICENSE) | | smmap | 5.0.2 | BSD | [License](https://github.com/gitpython-developers/smmap/blob/main/LICENSE) | | smart-open | 7.5.0 | MIT | [License](https://github.com/getcrest/smart-open/blob/dev/LICENSE) | @@ -49,16 +49,15 @@ This document lists third-party open source and other software packages used in | sentry-sdk | 2.51.0 | MIT | [License](https://github.com/getsentry/sentry-python/blob/master/LICENSE) | | sentence-transformers | 5.2.2 | Apache 2.0 | [License](https://github.com/huggingface/sentence-transformers/blob/main/LICENSE) | | scikit-learn | 1.8.0 | BSD | [License](https://github.com/scikit-learn/scikit-learn/blob/main/COPYING) | -| safetensors | 0.7.0 | Apache 2.0 | [License](https://github.com/huggingface/safetensors/blob/main/LICENSE) | +| safetensors | 0.8.0 | Apache 2.0 | [License](https://github.com/huggingface/safetensors/blob/main/LICENSE) | | safehttpx | 0.1.7 | MIT | [License](https://github.com/gradio-app/safehttpx/blob/main/LICENSE) | | sacrebleu | 2.6.0 | Apache 2.0 | [License](https://github.com/mjpost/sacrebleu/blob/master/LICENSE.txt) | | ruff | 0.14.14 | MIT | [License](https://github.com/astral-sh/ruff/blob/main/LICENSE) | | rpds-py | 0.30.0 | MIT | [License](https://raw.githubusercontent.com/crate-py/rpds/main/LICENSE) | | regex | 2026.1.15 | Apache 2.0 | [License](https://spdx.org/licenses/Apache-2.0.html) | -| regex | 2025.9.18 | Apache 2.0 | [License](https://spdx.org/licenses/Apache-2.0.html) | | referencing | 0.37.0 | MIT | [License](https://github.com/python-jsonschema/referencing/blob/main/COPYING) | | rank-bm25 | 0.2.2 | Apache 2.0 | [License](https://raw.githubusercontent.com/dorianbrown/rank_bm25/master/LICENSE) | -| pytorch | 2.10.0 | BSD | [License](https://github.com/pytorch/pytorch/blob/main/LICENSE) | +| pytorch | 2.13.0 | BSD | [License](https://github.com/pytorch/pytorch/blob/main/LICENSE) | | pythonjedi | 0.19.2 | MIT | [License](https://github.com/davidhalter/jedi/blob/master/LICENSE.txt) | | python3-rich | 14.3.2 | MIT | [License](https://github.com/Textualize/rich/blob/master/LICENSE) | | python3-charset-normalizer | 3.4.4 | MIT | [License](https://github.com/jawah/charset_normalizer/blob/master/LICENSE) | @@ -66,10 +65,10 @@ This document lists third-party open source and other software packages used in | python-semanticversion | 2.10.0 | BSD | [License](https://pypi.org/project/semantic-version/) | | python-protobuf | 6.33.5 | BSD | [License](https://raw.githubusercontent.com/protocolbuffers/protobuf/main/LICENSE) | | python-pluggy | 1.6.0 | MIT | [License](https://pypi.org/project/pluggy/) | -| python-multipart | 0.0.22 | Apache 2.0 | [License](https://pypi.org/project/python-multipart/) | -| python-jsonschema | 4.26.0 | MIT | [License](https://github.com/python-jsonschema/jsonschema/blob/main/LICENSE) | +| python-multipart | 0.0.32 | Apache 2.0 | [License](https://pypi.org/project/python-multipart/) | +| python-jsonschema | 4.23.0 | MIT | [License](https://github.com/python-jsonschema/jsonschema/blob/main/LICENSE) | | python-hpack | 4.1.0 | MIT | [License](https://raw.githubusercontent.com/python-hyper/hpack/master/LICENSE) | -| python-dotenv | 1.2.1 | BSD | [License](https://github.com/theskumar/python-dotenv/blob/main/LICENSE) | +| python-dotenv | 1.2.2 | BSD | [License](https://github.com/theskumar/python-dotenv/blob/main/LICENSE) | | python-distlib | 0.4.0 | Python Software Foundation License 2.0 | [License](https://github.com/pypa/distlib/blob/master/LICENSE.txt) | | python-dill | 0.3.8 | BSD | [License](https://github.com/uqfoundation/dill/blob/master/LICENSE) | | python-decorator | 5.2.1 | BSD | [License](https://github.com/micheles/decorator/blob/master/LICENSE) | @@ -82,7 +81,7 @@ This document lists third-party open source and other software packages used in | pytest-xdist | 3.8.0 | MIT | [License](https://github.com/pytest-dev/pytest-xdist/blob/master/LICENSE) | | pytest-timeout | 2.4.0 | MIT | [License](https://github.com/pytest-dev/pytest-timeout/blob/main/LICENSE) | | pytest-cov | 7.0.0 | MIT | [License](https://github.com/pytest-dev/pytest-cov/blob/master/LICENSE) | -| pytest | 9.0.2 | MIT | [License](https://github.com/pytest-dev/pytest/blob/main/LICENSE) | +| pytest | 9.0.3 | MIT | [License](https://github.com/pytest-dev/pytest/blob/main/LICENSE) | | pypi/setuptools | 80.10.2 | MIT | [License](https://github.com/pypa/setuptools/blob/main/LICENSE) | | pypa/sampleproject | 0.0.32 | MIT | [License](https://github.com/pypa/sampleproject/blob/main/LICENSE.txt) | | pyinvoke | 2.2.1 | BSD | [License](https://github.com/pyinvoke/invoke/blob/main/LICENSE) | @@ -92,11 +91,11 @@ This document lists third-party open source and other software packages used in | pydantic | 2.12.5 | MIT | [License](https://github.com/pydantic/pydantic/blob/main/LICENSE) | | pycparser | 3.0 | BSD | [License](https://github.com/eliben/pycparser/blob/master/LICENSE) | | pyca/pynacl | 1.6.2 | Apache 2.0 | [License](https://raw.githubusercontent.com/pyca/pynacl/main/LICENSE) | -| pyca/cryptography | 42.0.8 | BSD | [License](https://raw.githubusercontent.com/pyca/cryptography/main/LICENSE) | -| pyca/cryptography | 42.0.8 | Apache 2.0 | [License](https://raw.githubusercontent.com/pyca/cryptography/main/LICENSE) | +| pyca/cryptography | 49.0.0 | BSD | [License](https://raw.githubusercontent.com/pyca/cryptography/main/LICENSE) | +| pyca/cryptography | 49.0.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/pyca/cryptography/main/LICENSE) | | pyasn1-modules | 0.4.2 | BSD | [License](https://raw.githubusercontent.com/pyasn1/pyasn1-modules/master/LICENSE.txt) | -| pyasn1 | 0.6.2 | BSD | [License](https://raw.githubusercontent.com/pyasn1/pyasn1/master/LICENSE.txt) | -| pyarrow | 23.0.0 | Apache 2.0 | [License](https://github.com/apache/arrow/blob/main/LICENSE) | +| pyasn1 | 0.6.4 | BSD | [License](https://raw.githubusercontent.com/pyasn1/pyasn1/master/LICENSE.txt) | +| pyarrow | 25.0.0 | Apache 2.0 | [License](https://github.com/apache/arrow/blob/main/LICENSE) | | py3-google-auth | 2.48.0 | Apache 2.0 | [License](https://github.com/googleapis/google-auth-library-python/blob/main/LICENSE) | | py-spy | 0.4.1 | MIT | [License](https://raw.githubusercontent.com/benfred/py-spy/master/LICENSE) | | py-filelock | 3.20.3 | MIT | [License](https://py-filelock.readthedocs.io/en/latest/license.html) | @@ -108,7 +107,7 @@ This document lists third-party open source and other software packages used in | propcache | 0.4.1 | Apache 2.0 | [License](https://github.com/aio-libs/propcache/blob/master/LICENSE) | | prompt-toolkit/python-prompt-toolkit | 3.0.52 | BSD | [License](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/main/LICENSE) | | prometheus-fastapi-instrumentator | 7.0.0 | ISC (Internet Software Consortium) | [License](https://raw.githubusercontent.com/trallnag/prometheus-fastapi-instrumentator/master/LICENSE) | -| projectray | 2.53.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/ray-project/ray/master/LICENSE) | +| projectray | 2.56.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/ray-project/ray/master/LICENSE) | | pre-commit/pre-commit | 4.5.1 | MIT | [License](https://github.com/pre-commit/pre-commit/blob/main/LICENSE) | | portalocker | 3.2.0 | BSD | [License](https://github.com/wolph/portalocker/blob/develop/LICENSE) | | platformdirs | 4.5.1 | MIT | [License](https://raw.githubusercontent.com/tox-dev/platformdirs/main/LICENSE) | @@ -118,13 +117,13 @@ This document lists third-party open source and other software packages used in | parso | 0.8.5 | MIT | [License](https://github.com/davidhalter/parso/blob/main/LICENSE) | | paramiko | 4.0.0 | LGPL (Library or Lesser GPL) | [License](https://github.com/paramiko/paramiko/blob/main/LICENSE) | | pandas-python | 3.0.0 | BSD | [License](https://github.com/pandas-dev/pandas/blob/main/LICENSE) | -| orjson | 3.11.3 | Apache 2.0 | [License](https://github.com/ijl/orjson/blob/master/LICENSE-APACHE,https://github.com/ijl/orjson/blob/master/LICENSE-MIT) | +| orjson | 3.11.7 | Apache 2.0 | [License](https://github.com/ijl/orjson/blob/master/LICENSE-APACHE,https://github.com/ijl/orjson/blob/master/LICENSE-MIT) | | opentelemetry-semantic-conventions | 0.60b1 | Apache 2.0 | [License](https://raw.githubusercontent.com/open-telemetry/opentelemetry-python/main/LICENSE) | | opentelemetry-sdk | 1.39.1 | Apache 2.0 | [License](https://raw.githubusercontent.com/open-telemetry/opentelemetry-python/refs/heads/main/LICENSE) | | opentelemetry-python | 1.39.1 | Apache 2.0 | [License](https://github.com/open-telemetry/opentelemetry-python/blob/main/LICENSE) | | opentelemetry packages | 0.60b1 | Apache 2.0 | [License](https://github.com/open-telemetry/opentelemetry-python/blob/main/LICENSE) | | opencensus | 0.11.4 | Apache 2.0 | [License](https://github.com/census-instrumentation/opencensus-python/blob/master/LICENSE) | -| openai | 2.16.0 | Apache 2.0 | [License](https://github.com/openai/openai-python/blob/main/LICENSE) | +| openai | 2.24.0 | Apache 2.0 | [License](https://github.com/openai/openai-python/blob/main/LICENSE) | | omegaconf | 2.3.0 | BSD | [License](https://github.com/omry/omegaconf/blob/master/LICENSE) | | nvidia-nvtx-cu12 | 12.8.90 | Apache 2.0 | [License](https://pypi.org/project/nvidia-nvtx-cu12/) | | nvidia-nvshmem-cu12 | 3.4.5 | Nvidia Proprietary License | [License](https://raw.githubusercontent.com/NVIDIA/nvshmem/devel/License.txt) | @@ -142,24 +141,24 @@ This document lists third-party open source and other software packages used in | nvidia-cuda-cupti-cu12 | 12.8.90 | Nvidia Proprietary License | [License](https://docs.nvidia.com/cuda/eula/index.html) | | nvidia-cublas-cu12 | 12.8.4.1 | Nvidia Proprietary License | [License](https://docs.nvidia.com/cuda/eula/index.html) | | nodeenv | 1.10.0 | BSD | [License](https://github.com/ekalinin/nodeenv/blob/master/LICENSE) | -| nemo-run | 0.7.0 | Apache 2.0 | [License](https://github.com/NVIDIA/NeMo-Run/blob/main/LICENSE) | +| nemo-run | 0.9.0rc0.dev0 | Apache 2.0 | [License](https://github.com/NVIDIA/NeMo-Run/blob/main/LICENSE) | | nemo-evaluator-launcher | 0.1.46 | Apache 2.0 | [License](https://pypi.org/project/cuda-bindings/) | | mypy-extensions | 1.1.0 | MIT | [License](https://github.com/python/mypy_extensions/blob/master/LICENSE) | | mypy | 1.19.1 | MIT | [License](https://pypi.org/project/mypy/) | | multipledispatch | 1.0.0 | BSD | [License](https://github.com/mrocklin/multipledispatch/blob/master/LICENSE.txt) | | multidict | 6.7.1 | Apache 2.0 | [License](https://raw.githubusercontent.com/aio-libs/multidict/master/LICENSE) | | multi_process | 0.70.16 | BSD | [License](https://github.com/uqfoundation/multiprocess/blob/master/LICENSE) | -| msgpack | 1.1.2 | Apache 2.0 | [License](https://raw.githubusercontent.com/msgpack/msgpack-python/main/COPYING) | +| msgpack | 1.2.1 | Apache 2.0 | [License](https://raw.githubusercontent.com/msgpack/msgpack-python/main/COPYING) | | mpmath | 1.3.0 | BSD | [License](https://github.com/fredrik-johansson/mpmath/blob/master/LICENSE) | | mdurl | 0.1.2 | MIT | [License](https://raw.githubusercontent.com/executablebooks/mdurl/master/LICENSE) | -| mcp | 1.26.0 | MIT | [License](https://github.com/modelcontextprotocol/python-sdk/blob/main/LICENSE) | +| mcp | 1.28.1 | MIT | [License](https://github.com/modelcontextprotocol/python-sdk/blob/main/LICENSE) | | matplotlib-inline | 0.2.1 | BSD | [License](https://github.com/ipython/matplotlib-inline/blob/master/LICENSE) | | math-verify | 0.9.0 | Apache 2.0 | [License](https://github.com/huggingface/math-verify/blob/main/LICENCE) | | markdown-it-py | 4.0.0 | MIT | [License](https://github.com/executablebooks/markdown-it-py/blob/master/LICENSE) | | marisa-trie | 1.3.1 | MIT | [License](https://github.com/pytries/marisa-trie/blob/master/LICENSE) | -| lxml | 6.0.2 | BSD | [License](https://github.com/lxml/lxml/blob/main/LICENSE) | +| lxml | 6.1.1 | BSD | [License](https://github.com/lxml/lxml/blob/main/LICENSE) | | loguru | 0.7.3 | MIT | [License](https://github.com/Delgan/loguru/blob/master/LICENSE) | -| litellm | 1.81.6 | MIT | [License](https://github.com/BerriAI/litellm/blob/main/LICENSE) | +| litellm | 1.84.10 | MIT | [License](https://github.com/BerriAI/litellm/blob/main/LICENSE) | | librt | 0.7.8 | MIT | [License](https://github.com/mypyc/librt/blob/main/LICENSE) | | libcst | 1.8.6 | MIT | [License](https://github.com/Instagram/LibCST/blob/main/LICENSE) | | leptonai | 0.27.0 | Apache 2.0 | [License](https://sourceforge.net/projects/lepton-ai.mirror/) | @@ -176,22 +175,22 @@ This document lists third-party open source and other software packages used in | iso639-lang | 2.6.3 | MIT | [License](https://pypi.org/project/iso639-lang/) | | ipython-pygments-lexers | 1.1.1 | BSD | [License](https://github.com/ipython/ipython-pygments-lexers/blob/main/LICENSE) | | inquirerpy | 0.3.4 | MIT | [License](https://github.com/kazhala/InquirerPy/blob/master/LICENSE) | -| importlib_metadata | 8.7.1 | Apache 2.0 | [License](https://pypi.org/project/importlib-metadata/) | +| importlib_metadata | 8.5.0 | Apache 2.0 | [License](https://pypi.org/project/importlib-metadata/) | | idna | 3.11 | BSD | [License](https://github.com/kjd/idna/blob/master/LICENSE.md) | | identify | 2.6.16 | MIT | [License](https://github.com/pre-commit/identify/blob/main/LICENSE) | | hyperframe | 6.1.0 | MIT | [License](https://github.com/python-hyper/hyperframe/blob/master/LICENSE) | | hydra-core | 1.3.2 | MIT | [License](https://github.com/facebookresearch/hydra/blob/main/LICENSE) | -| huggingface-hub | 1.3.7 | Apache 2.0 | [License](https://github.com/huggingface/huggingface_hub/blob/main/LICENSE) | +| huggingface-hub | 1.16.1 | Apache 2.0 | [License](https://github.com/huggingface/huggingface_hub/blob/main/LICENSE) | | httpx-sse | 0.4.3 | MIT | [License](https://raw.githubusercontent.com/florimondmanca/httpx-sse/master/LICENSE) | -| httpx | 0.27.2 | BSD | [License](https://github.com/encode/httpx/blob/master/LICENSE.md) | +| httpx | 0.28.1 | BSD | [License](https://github.com/encode/httpx/blob/master/LICENSE.md) | | httpcore | 1.0.9 | BSD | [License](https://raw.githubusercontent.com/encode/httpcore/master/LICENSE.md) | -| hf-xet | 1.2.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/huggingface/xet-core/main/LICENSE) | +| hf-xet | 1.5.1 | Apache 2.0 | [License](https://raw.githubusercontent.com/huggingface/xet-core/main/LICENSE) | | h2 | 4.3.0 | MIT | [License](https://github.com/python-hyper/h2/blob/master/LICENSE) | | h11 | 0.16.0 | MIT | [License](https://github.com/python-hyper/h11/blob/master/LICENSE.txt) | | grpc | 1.76.0 | Apache 2.0 | [License](https://github.com/grpc/grpc/blob/master/LICENSE) | | groovy | 0.1.2 | MIT | [License](https://github.com/abidlabs/groovy/blob/main/LICENSE) | -| gradio-client | 2.0.3 | Apache 2.0 | [License](https://github.com/gradio-app/gradio/blob/main/LICENSE) | -| gradio | 6.5.1 | Apache 2.0 | [License](https://github.com/gradio-app/gradio/blob/main/LICENSE) | +| gradio-client | 2.5.0 | Apache 2.0 | [License](https://github.com/gradio-app/gradio/blob/main/LICENSE) | +| gradio | 6.20.0 | Apache 2.0 | [License](https://github.com/gradio-app/gradio/blob/main/LICENSE) | | googleapis-common-protos | 1.72.0 | Apache 2.0 | [License](https://github.com/gradio-app/gradio/blob/main/LICENSE) | | google/fiddle | 0.3.0 | Apache 2.0 | [License](https://github.com/gradio-app/gradio/blob/main/LICENSE) | | google-api-core | 2.29.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/googleapis/python-api-common-protos/main/LICENSE) | @@ -210,7 +209,7 @@ This document lists third-party open source and other software packages used in | executing | 2.2.1 | MIT | [License](https://github.com/alexmojaki/executing/blob/master/LICENSE) | | execnet | 2.1.2 | MIT | [License](https://pypi.org/project/execnet/) | | exceptiongroup | 1.3.1 | MIT | [License](https://github.com/agronholm/exceptiongroup/blob/main/LICENSE) | -| evalplus | 0.3.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/evalplus/evalplus/refs/heads/master/LICENSE) | +| evalplus | 0.3.0.dev27 | Apache 2.0 | [License](https://raw.githubusercontent.com/evalplus/evalplus/refs/heads/master/LICENSE) | | encode/uvicorn | 0.40.0 | BSD | [License](https://github.com/encode/uvicorn/blob/master/LICENSE.md) | | edit-distance | 0.8.1 | MIT | [License](https://github.com/roy-ht/editdistance/blob/master/LICENSE) | | docstring-parser | 0.17.0 | MIT | [License](https://raw.githubusercontent.com/rr-/docstring_parser/master/LICENSE.md) | @@ -218,8 +217,8 @@ This document lists third-party open source and other software packages used in | distro | 1.9.0 | Apache 2.0 | [License](https://github.com/python-distro/distro/blob/master/LICENSE) | | diskcache | 5.6.3 | Apache 2.0 | [License](http://www.apache.org/licenses/LICENSE-2.0) | | datasets | 3.6.0 | Apache 2.0 | [License](https://github.com/huggingface/datasets/blob/main/LICENSE) | -| cuda-pathfinder | 1.3.3 | Apache 2.0 | [License](https://www.apache.org/licenses/LICENSE-2.0) | -| cuda-bindings | 12.9.4 | NVIDIA CUDA Toolkit License Agreement | [License](https://pypi.org/project/cuda-bindings/) | +| cuda-pathfinder | 1.5.6 | Apache 2.0 | [License](https://www.apache.org/licenses/LICENSE-2.0) | +| cuda-bindings | 13.3.1 | NVIDIA CUDA Toolkit License Agreement | [License](https://pypi.org/project/cuda-bindings/) | | contextlib2 | 21.6.0 | Apache 2.0 | [License](https://policies.python.org/pypi.org/Acceptable-Use-Policy/#credits-license) | | colorama | 0.4.6 | BSD | [License](https://raw.githubusercontent.com/tartley/colorama/master/LICENSE.txt) | | cloudpickle | 3.0.0 | BSD | [License](https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE) | @@ -239,8 +238,8 @@ This document lists third-party open source and other software packages used in | annotated-doc | 0.0.4 | MIT | [License](https://github.com/fastapi/annotated-doc/blob/main/LICENSE) | | aiosignal | 1.4.0 | Apache 2.0 | [License](https://github.com/aio-libs/aiosignal/blob/master/LICENSE) | | aiohttp-cors | 0.8.1 | Apache 2.0 | [License](https://github.com/aio-libs/aiohttp-cors/blob/master/LICENSE) | -| aiohttp | 3.13.0 | MIT | [License](https://github.com/aio-libs/aiohttp/blob/main/LICENSE) | -| aiohttp | 3.13.0 | Apache 2.0 | [License](https://github.com/aio-libs/aiohttp/blob/main/LICENSE) | +| aiohttp | 3.14.1 | MIT | [License](https://github.com/aio-libs/aiohttp/blob/main/LICENSE) | +| aiohttp | 3.14.1 | Apache 2.0 | [License](https://github.com/aio-libs/aiohttp/blob/main/LICENSE) | | aiohappyeyeballs | 2.6.1 | Python Software Foundation License 2.0 | [License](https://github.com/aio-libs/aiohappyeyeballs/blob/main/LICENSE) | | aiofiles | 24.1.0 | Apache 2.0 | [License](https://github.com/Tinche/aiofiles/blob/main/LICENSE) | | absl-py | 2.4.0 | Apache 2.0 | [License](https://raw.githubusercontent.com/abseil/abseil-py/master/LICENSE) | @@ -249,20 +248,20 @@ This document lists third-party open source and other software packages used in | Traitlets | 5.14.3 | BSD | [License](https://github.com/ipython/traitlets/blob/main/LICENSE) | | TOML | 0.10.2 | MIT | [License](https://github.com/uiri/toml/blob/master/LICENSE) | | SymPy | 1.14.0 | BSD | [License](https://github.com/sympy/sympy/blob/main/LICENSE) | -| Starlette | 0.50.0 | BSD | [License](https://github.com/Kludex/starlette/blob/main/LICENSE.md) | +| Starlette | 1.3.1 | BSD | [License](https://github.com/Kludex/starlette/blob/main/LICENSE.md) | | SciPy | 1.17.0 | BSD | [License](https://github.com/scipy/scipy/blob/main/LICENSE.txt) | | RonnyPfannschmidt/iniconfig | 2.3.0 | MIT | [License](https://github.com/pytest-dev/iniconfig/blob/main/LICENSE) | | Python-RSA | 4.9.1 | Apache 2.0 | [License](https://github.com/sybrenstuvel/python-rsa/blob/main/LICENSE) | -| Python tzdata | 2025.2 | Apache 2.0 | [License](https://github.com/python/tzdata/blob/master/LICENSE) | +| Python tzdata | 2025.3 | Apache 2.0 | [License](https://github.com/python/tzdata/blob/master/LICENSE) | | Python six | 1.17.0 | MIT | [License](https://github.com/benjaminp/six/blob/master/LICENSE) | -| Pygments - Python syntax highlighter | 2.19.2 | BSD | [License](https://github.com/pygments/pygments/blob/master/LICENSE) | +| Pygments - Python syntax highlighter | 2.20.0 | BSD | [License](https://github.com/pygments/pygments/blob/master/LICENSE) | | PyYAML | 6.0.3 | MIT | [License](https://github.com/yaml/pyyaml/blob/main/LICENSE) | | PyTZ - Python Time Zone Library | 2025.2 | MIT | [License](https://github.com/stub42/pytz/blob/master/LICENSE.txt) | -| PyJWT | 2.11.0 | MIT | [License](https://spdx.org/licenses/MIT.html) | -| PillowPython | 12.1.0 | MIT | [License](https://github.com/python-pillow/Pillow/blob/main/LICENSE) | +| PyJWT | 2.13.0 | MIT | [License](https://spdx.org/licenses/MIT.html) | +| PillowPython | 12.3.0 | MIT | [License](https://github.com/python-pillow/Pillow/blob/main/LICENSE) | | Packaging | 26.0 | BSD | [License](https://github.com/pypa/packaging/blob/main/LICENSE) | | Packaging | 26.0 | Apache 2.0 | [License](https://github.com/pypa/packaging/blob/main/LICENSE) | -| OpenCensus | 0.1.3 | Apache 2.0 | [License](https://github.com/census-instrumentation/opencensus-python/blob/master/LICENSE) | +| opencensus-context | 0.1.3 | Apache 2.0 | [License](https://github.com/census-instrumentation/opencensus-python/blob/master/LICENSE) | | NumPy | 2.1.3 | BSD | [License](https://github.com/numpy/numpy/blob/main/LICENSE.txt) | | NetworkX | 3.6.1 | BSD | [License](https://github.com/networkx/networkx/blob/master/LICENSE.txt) | | MarkupSafe | 3.0.3 | BSD | [License](https://github.com/pallets/markupsafe/blob/main/LICENSE.txt) | @@ -271,6 +270,6 @@ This document lists third-party open source and other software packages used in | Flask | 3.1.2 | BSD | [License](https://raw.githubusercontent.com/pallets/flask/main/LICENSE.txt) | | Deprecated | 1.3.1 | MIT | [License](https://raw.githubusercontent.com/laurent-laporte-pro/deprecated/master/LICENSE.rst) | | Coverage | 7.13.2 | Apache 2.0 | [License](https://github.com/nedbat/coveragepy/blob/master/LICENSE.txt) | -| Click - Python Command Line Utility | 8.1.8 | BSD | [License](https://click.palletsprojects.com/en/stable/license/) | +| Click - Python Command Line Utility | 8.4.2 | BSD | [License](https://click.palletsprojects.com/en/stable/license/) | | BeautifulSoup4 | 4.14.3 | MIT | [License](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) | | python-hyper/h2 | 4.3.0 | MIT | [License](https://github.com/python-hyper/h2/blob/master/LICENSE) | diff --git a/NOTICE b/NOTICE index e9e43ba..fdb7efe 100644 --- a/NOTICE +++ b/NOTICE @@ -19,7 +19,7 @@ For full license texts, see the links below or the documentation of each component. - nemo-skills (NVIDIA NeMo-Skills) - https://github.com/NVIDIA/NeMo-Skills + https://github.com/NVIDIA-NeMo/Skills License: Apache-2.0 - omegaconf diff --git a/README.md b/README.md index 2640664..59de1dc 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,8 @@ uv run pre-commit install > ⚠️ **Developers:** Always run `uv run pre-commit install` after cloning. This enables automatic code quality checks on every commit. +> πŸ”Œ **Air-gapped / no internet on the install host?** Skip the local install and drive `nflow` from the prebuilt `nvflow-client` container (CLI + venv baked in, no `uv sync`). See [docs/remote-launch.md](docs/remote-launch.md). + ### Activating the Virtual Environment (Optional) By default, use `uv run ` to run commands in the project's virtual environment. If you prefer to activate the environment directly: @@ -123,11 +125,13 @@ pytest ## πŸ”§ Cluster Setup -To run workflows on a Slurm cluster you need to: (1) build the four NVFlow -container images from the Dockerfiles in [`dockerfiles/`](dockerfiles/), -(2) convert them to `.sqsh` for Slurm, and (3) write a cluster config -(`cluster_configs/my_cluster.yaml`). The containers are self-sufficient β€” -all dependencies are pre-installed, so no runtime downloads are needed. +To run workflows on a Slurm cluster you need to: (1) build the NVFlow +container images from the Dockerfiles in [`dockerfiles/`](dockerfiles/) +(`nemo-rl`, `nemo-gym`, `nemo-skills`, `vllm`, `vllm-grpo`; only `sglang` is +pulled as-is), (2) convert them to `.sqsh` for Slurm, and (3) write a cluster +config (`cluster_configs/my_cluster.yaml`). Every image bakes the packages and +virtual environments its stages need, so no stage resolves dependencies at job +runtime (see [INSTALL.md](INSTALL.md)). > **See [INSTALL.md](INSTALL.md)** for the complete setup guide > (build, sanity-check, `.sqsh` conversion, model staging, cluster @@ -142,6 +146,9 @@ Once cluster setup is complete, set the config directory: export NEMO_SKILLS_CONFIG_DIR=/path/to/nvflow/cluster_configs ``` +> **Airgapped / no local install?** Drive `nflow` from the `nvflow-client` +> container (no `uv sync` needed) over an SSH tunnel β€” see [docs/remote-launch.md](docs/remote-launch.md). + ## πŸš€ Quick Start ### CLI Invocation @@ -200,16 +207,14 @@ nvflow/ β”‚ β”œβ”€β”€ stages/sdg/ # Example SDG stage β”‚ β”œβ”€β”€ prompts/ # Prompt templates β”‚ └── workflows/ # Example workflows - └── finance/ # Finance reasoning recipe - β”œβ”€β”€ stages/ # Stage implementations - β”‚ β”œβ”€β”€ download/ # SEC filing download - β”‚ β”œβ”€β”€ evaluation/ # Evaluation stages - β”‚ β”œβ”€β”€ rl/ # GRPO RL training stages - β”‚ β”œβ”€β”€ sdg/ # SDG stages - β”‚ β”œβ”€β”€ sft/ # SFT training stages - β”‚ └── shared/ # Shared stages (data_transformation, train_validation_split) - β”œβ”€β”€ prompts/ # Prompt templates - └── workflows/ # Workflow configs + β”œβ”€β”€ finance/ # Finance reasoning recipe + β”‚ β”œβ”€β”€ stages/ # Stage implementations + β”‚ β”œβ”€β”€ prompts/ # Prompt templates + β”‚ └── workflows/ # Workflow configs + └── multimodal/ # Multimodal HopChain recipe + β”œβ”€β”€ stages/ # Image filtering and SDG stages + β”œβ”€β”€ prompts/ # Vision-language prompt templates + └── workflows/ # HopChain workflow configs ``` ## πŸ“ Creating a Stage @@ -261,7 +266,7 @@ End-to-end pipeline for generating synthetic financial Q&A data from SEC filings **Quick Links:** - [Quick Start (~3 hour demo)](docs/recipes/finance/quick-start.md) - Get started quickly with 7 companies - [Workflow Guides](docs/recipes/finance/workflows/) - Detailed guides for all 6 workflows -- [Stage Reference](docs/recipes/finance/stages/) - Technical specifications for 42 stages +- [Stage Reference](docs/recipes/finance/stages/) - Technical specifications for all 37 stages **Pipeline:** ``` @@ -269,12 +274,31 @@ download-sec β†’ template-sdg / document-sdg β†’ sft β†’ eval β†’ grpo ``` **Features:** -- 42 stages across 6 workflows +- 37 stages across 6 workflows (1 + 6 + 7 + 6 + 7 + 10) - Two SDG approaches (template-based & document-grounded) - Multiple model support (GPT-OSS-120B, Qwen3, Nemotron) -- Produces 80K+ synthetic Q&A pairs +- Produces 300K+ synthetic Q&A pairs - Complete training and evaluation pipeline +### Multimodal HopChain Recipe + +**πŸ“š [Complete Multimodal Recipe Documentation β†’](docs/recipes/multimodal/README.md)** + +HopChain-inspired multimodal synthetic data generation for multi-hop +vision-language reasoning. + +**Pipeline:** +``` +image-filter β†’ identify-categories β†’ localize-instances β†’ sample-combinations + β†’ generate-queries β†’ verify β†’ judge/reconcile β†’ difficulty-filter β†’ sft-traces +``` + +**Features:** +- Two workflows: image filtering and SDG +- SAM-backed instance localization +- Structural verification and optional external LLM judges +- Optional SFT reasoning-trace generation and filtering + ## πŸ“š CLI Commands ```bash @@ -314,5 +338,5 @@ Apache-2.0 ## πŸ™ Acknowledgments Built on: -- [NeMo-Skills](https://github.com/NVIDIA/NeMo-Skills) +- [NeMo-Skills](https://github.com/NVIDIA-NeMo/Skills) - [NeMo-RL](https://github.com/NVIDIA-NeMo/RL) diff --git a/cluster_configs/containers.yaml b/cluster_configs/containers.yaml index 14281b9..6110e94 100644 --- a/cluster_configs/containers.yaml +++ b/cluster_configs/containers.yaml @@ -11,46 +11,73 @@ # sbatch --account= scripts/setup_containers.sh --config cluster_configs/my_containers.yaml ./containers # # Naming: -# YAML KEYS (nemo-skills, nemo-rl, vllm, vllm-grpo, sglang) match the +# YAML KEYS (nemo-rl, nemo-gym, nemo-skills, vllm, vllm-grpo, sglang) match the # short names the workflow code uses to look up containers -- do NOT rename # them. Only the values (registry/tag refs) change between deployments. # # Format: -# - Simple string: image reference (e.g., your-registry/nvflow-nemo-skills:0229040) +# - Simple string: image reference (e.g., your-registry/nvflow-nemo-skills:v1.1.2) # - Nested object: Platform-specific tags (amd64/arm64 keys) # -# Self-sufficient containers (all deps pre-installed, no runtime downloads): -# nemo-rl -> dockerfiles/Dockerfile.nemo-rl (base: nvcr.io/nvidia/nemo-rl:v0.6.0) -# nemo-skills -> dockerfiles/Dockerfile.nemo-skills (base: ubuntu:22.04, NeMo-Skills @ 0229040) -# vllm -> dockerfiles/Dockerfile.vllm (base: vllm/vllm-openai:v0.18.1) -# vllm-grpo -> dockerfiles/Dockerfile.vllm-grpo (base: vllm/vllm-openai:v0.17.1) -# sglang -> pulled as-is from Docker Hub (no custom Dockerfile) +# Built locally (custom images; all deps pre-installed, no runtime downloads): +# nemo-rl -> dockerfiles/Dockerfile.nemo-rl (base: nvcr.io/nvidia/nemo-rl:v0.7.0) +# nemo-skills -> dockerfiles/Dockerfile.nemo-skills (base: ubuntu:22.04, NeMo-Skills @ e06c9b90) +# vllm -> dockerfiles/Dockerfile.vllm (base: vllm/vllm-openai:v0.22.0) +# vllm-grpo -> dockerfiles/Dockerfile.vllm --build-arg VLLM_VERSION=v0.20.0 +# nemo-gym -> dockerfiles/Dockerfile.nemo-gym (base: python:3.12-slim, CPU-only Gym client) +# Pulled as-is (no custom Dockerfile): +# sglang -> lmsysorg/sglang:v0.5.10.post1 +# +# Optional launcher image for airgapped users (airgap-only; NOT a worker -- see +# the nvflow-client entry below): +# nvflow-client -> dockerfiles/Dockerfile.nvflow (base: digest-pinned ubuntu:24.04; multi-arch amd64+arm64) containers: # --------------------------------------------------------------------------- # Required: Built locally from dockerfiles/, then pushed to your registry # (see INSTALL.md Step 1 and Step 2). # --------------------------------------------------------------------------- - # Tested: nvflow-nemo-rl:v0.6.0 (extends nvcr.io/nvidia/nemo-rl:v0.6.0) - nemo-rl: /nvflow-nemo-rl:v0.6.0 + # SFT/GRPO trainer. Extends the NeMo-RL v0.7.0 base with the NeMo-Gym + # per-component venvs baked in, so no `uv` resolve happens at job runtime. + # dockerfiles/Dockerfile.nemo-rl (upstream Gym main 33ef60369). The stock base + # also works if your compute nodes have internet during training. + nemo-rl: /nvflow-nemo-rl:v0.7.0 + + # CPU-only Gym client for the Gym-only stages (DG-SDG gym stages + GRPO + # prepare_data/prefetch_cache). dockerfiles/Dockerfile.nemo-gym (base: + # python:3.12-slim; upstream Gym main 33ef60369; per-component venvs baked). + nemo-gym: /nvflow-nemo-gym:0.4.0 - # Tested: nvflow-nemo-skills:0229040 (NeMo-Skills @ commit 0229040) - nemo-skills: /nvflow-nemo-skills:0229040 + # Tested: nvflow-nemo-skills:v1.1.2 (NeMo-Skills @ commit e06c9b90) + nemo-skills: /nvflow-nemo-skills:v1.1.2 - # Tested: nvflow-vllm:v0.18.1 (extends vllm/vllm-openai:v0.18.1; pre-cached - # tiktoken + openai_harmony; multi-arch amd64 + arm64) - vllm: /nvflow-vllm:v0.18.1 + # Tested: nvflow-vllm:v0.22.0 (extends vllm/vllm-openai:v0.22.0; + # pre-cached tiktoken + openai_harmony; multi-arch amd64 + arm64) + vllm: /nvflow-vllm:v0.22.0 - # Tested: nvflow-vllm-grpo:v0.17.1 (extends vllm/vllm-openai:v0.17.1; pinned - # to match NeMo-RL v0.6.0 colocated vLLM) - vllm-grpo: /nvflow-vllm-grpo:v0.17.1 + # Tested: nvflow-vllm:v0.20.0 (extends vllm/vllm-openai:v0.20.0; pinned to + # match NeMo-RL v0.7.0 colocated vLLM). Same repo as `vllm`, different tag. + vllm-grpo: /nvflow-vllm:v0.20.0 # --------------------------------------------------------------------------- - # Required: Pulled as-is from Docker Hub (no custom Dockerfile) + # Required: Pulled as-is (no custom Dockerfile) # --------------------------------------------------------------------------- # Tested: lmsysorg/sglang:v0.5.10.post1 sglang: lmsysorg/sglang:v0.5.10.post1 + # --------------------------------------------------------------------------- + # OPTIONAL -- launcher image, ONLY for users in an airgapped environment who + # cannot `uv sync` / pip-install. It is NOT a worker and is NOT referenced by + # my_cluster.yaml `containers:`; it bundles the `nflow` CLI + baked venv to + # drive NVFlow over an ssh_tunnel with no host install and no user-side internet. + # The default install remains `uv sync` (README / INSTALL.md). + # Built multi-arch (amd64 + arm64) from dockerfiles/Dockerfile.nvflow; see + # docs/remote-launch.md. + # + # Tested: nvflow-client:v1.1.2. Substitute the tag you built and pushed. + # --------------------------------------------------------------------------- + nvflow-client: /nvflow-client: + # --------------------------------------------------------------------------- # Optional: Not currently used by NVFlow recipes # Uncomment and update if needed for your workflows. diff --git a/cluster_configs/template-slurm.yaml b/cluster_configs/template-slurm.yaml index 86e14a3..387339a 100644 --- a/cluster_configs/template-slurm.yaml +++ b/cluster_configs/template-slurm.yaml @@ -5,14 +5,15 @@ # # Then update all values with your settings. # -# Container versions tested with this release (self-sufficient, no runtime downloads): -# nemo-skills: nvflow-nemo-skills (NeMo-Skills @ 0229040) -# vllm: nvflow-vllm (base vllm/vllm-openai v0.18.1, standalone SDG/eval) -# vllm-grpo: nvflow-vllm-grpo (base vllm/vllm-openai v0.17.1, GRPO rollouts/judge) +# Container versions tested with this release: +# nemo-skills: nvflow-nemo-skills (NeMo-Skills @ e06c9b90, tag v1.1.2) +# vllm: nvflow-vllm (base vllm/vllm-openai v0.22.0, standalone SDG/eval) +# vllm-grpo: nvflow-vllm at the v0.20.0 tag (base vllm/vllm-openai v0.20.0, GRPO rollouts/judge) # sglang: lmsysorg/sglang v0.5.10.post1 -# nemo-rl: nvflow-nemo-rl (base nvcr.io/nvidia/nemo-rl:v0.6.0, pre-built venvs + Gym) +# nemo-rl: nvflow-nemo-rl (base nvcr.io/nvidia/nemo-rl v0.7.0; SFT/GRPO trainer, Gym venvs baked) +# nemo-gym: nvflow-nemo-gym (base python:3.12-slim, CPU-only; per-component Gym venvs β€” DG-SDG gym stages) # -# Reference: https://github.com/NVIDIA/NeMo-Skills +# Reference: https://github.com/NVIDIA-NeMo/Skills executor: slurm @@ -80,11 +81,13 @@ extra_sandbox_args: # After converting containers to .sqsh format (see INSTALL.md), paste paths here. containers: # Required containers (self-sufficient β€” all deps pre-installed, no runtime downloads) - nemo-skills: /nvflow-nemo-skills.sqsh # Orchestration client (eval, SDG, data prep) - vllm: /nvflow-vllm.sqsh # vLLM v0.18.1 standalone (SDG, eval) - vllm-grpo: /nvflow-vllm-grpo.sqsh # vLLM v0.17.1 standalone (GRPO rollouts, judge) - sglang: /sglang.sqsh # sglang inference server (SDG stages 3-5) - nemo-rl: /nvflow-nemo-rl.sqsh # NeMo-RL v0.6.0 for SFT and GRPO training + # setup_containers.sh names each file -.sqsh. + nemo-skills: /nemo-skills-v1.1.2.sqsh # Orchestration client (eval, SDG, data prep) + vllm: /vllm-v0.22.0.sqsh # vLLM v0.22.0 standalone (SDG, eval) + vllm-grpo: /vllm-grpo-v0.20.0.sqsh # vLLM v0.20.0 standalone (GRPO rollouts, judge); same image repo as vllm, different tag + sglang: /sglang-v0.5.10.post1.sqsh # sglang inference server (SDG stages 3-5) + nemo-rl: /nemo-rl-v0.7.0.sqsh # NeMo-RL v0.7.0 for SFT and GRPO training + nemo-gym: /nemo-gym-0.4.0.sqsh # CPU-only Gym client for the Gym-only stages (DG-SDG gym stages; no NeMo-RL trainer) # Optional containers (not currently used by NVFlow recipes) # trtllm: /trtllm.sqsh # megatron: /megatron.sqsh @@ -97,17 +100,22 @@ containers: # Map cluster paths to container paths mounts: - :/hf_models # HuggingFace models - - :/workspace # Your workspace + # /workspace is a WRITABLE DATA dir (outputs + caches) -- NOT the source repo. + # Recipe code and checked-in assets (prompts, dataset descriptors, overlays) ship + # via the nemo-run packaged snapshot at /nemo_run/code on every worker, so the + # repo is never mounted. This mount only needs to hold writable runtime data: + # /workspace/outputs/** stage outputs, checkpoints, SEC cache, eval-datasets + # /workspace/cache/huggingface HF_HOME (see env_vars below) + # Point it at a dedicated data dir (e.g. a sibling of your repo checkout), not the + # checkout itself. + - :/workspace # Writable data dir (outputs + cache) # Add more mounts as needed: # - /lustre/data:/data # - # --- NeMo-RL / NeMo-Gym source overlays (dev mode only) --- - # The nvflow-nemo-rl container includes NeMo-RL source and pre-built Gym - # venvs. Mounting host clones here shadows the container's venvs and - # breaks GRPO stages. Only uncomment for local source iteration with - # NRL_FORCE_REBUILD_VENVS=true enabled below. - # - :/opt/NeMo-RL - # - :/opt/NeMo-RL/3rdparty/Gym-workspace/Gym + # No Gym source mount is needed: the nemo-rl and nemo-gym images both bake + # their NeMo-Gym venvs, so nothing is built at job runtime. Bind-mounting a + # host clone over /opt/nemo-rl/3rdparty/Gym-workspace/Gym shadows those baked + # venvs and breaks the GRPO stages -- do it only for dev-mode work. # ============================================================================= # Timeouts (per partition) @@ -135,13 +143,17 @@ env_vars: - VIRTUAL_ENV_PROMPT= # Unset venv prompt # --- Offline enforcement (recommended) --- - # Prevents accidental network calls at runtime. Containers are self-sufficient. - # Clear HF_*_OFFLINE temporarily for one-time stages that download external - # data (download_sec_filings, create_seed_data). UV_OFFLINE should stay set. + # Prevents accidental network calls at runtime. Clear HF_*_OFFLINE temporarily + # for one-time stages that download external data (download_sec_filings, + # create_seed_data). - HF_HUB_OFFLINE=1 - HF_DATASETS_OFFLINE=1 - TRANSFORMERS_OFFLINE=1 - - UV_OFFLINE=true + # UV_OFFLINE: left UNSET (global flag). The images bake every venv they need, + # so no stage resolves packages at runtime either way. Leaving it unset keeps + # the escape hatch for dev-mode work on components outside the baked set. + # Set true only for a strict-airgap cluster. + # - UV_OFFLINE=true # Pre-cached tiktoken encodings (baked into vllm/vllm-grpo containers; # set here for uniform coverage across all container types). @@ -153,10 +165,6 @@ env_vars: - MIN_WORKER_PORT=7000 - MAX_WORKER_PORT=8000 - # Only enable in dev mode when iterating on NeMo-RL/Gym source overlays. - # Forces Ray workers to rebuild venvs from mounted source (requires internet). - # - NRL_FORCE_REBUILD_VENVS=true - # API keys (keep these secret, don't commit to git!) # - HF_TOKEN= # - WANDB_API_KEY= diff --git a/dockerfiles/Dockerfile.nemo-gym b/dockerfiles/Dockerfile.nemo-gym new file mode 100644 index 0000000..9bb233c --- /dev/null +++ b/dockerfiles/Dockerfile.nemo-gym @@ -0,0 +1,102 @@ +# ============================================================================= +# NVFlow NeMo-Gym container (CPU-only). For Gym-only stages (prepare_data / +# prefetch_cache), the eval rollout client, and SDG rollouts. GRPO/training +# uses Dockerfile.nemo-rl. +# +# Bakes one venv per component (`gym env start ... +dry_run=true`) for +# equivalence_llm_judge, finance_sec_search and format_verification, reused +# offline at runtime. They live in /opt/gym-venvs, outside /opt/Gym, so a dev +# source mount overlays source only. Non-baked components fall back to an +# on-demand `uv` build, which needs a writable venv dir and network. +# +# Build (GYM_REF is a pinned SHA, so a cached clone layer cannot go stale). +# Single-arch, host platform; see docker_instructions.md for multi-arch: +# docker build -f dockerfiles/Dockerfile.nemo-gym -t nvflow-nemo-gym:0.4.0 . +# ============================================================================= +ARG PYTHON_VERSION=3.12 +ARG GYM_VENV_DIR=/opt/gym-venvs +ARG GYM_CLI_VENV=/opt/gym-cli-venv + +# --- builder ----------------------------------------------------------------- +FROM python:${PYTHON_VERSION}-slim AS builder +ARG PYTHON_VERSION +ARG GYM_VENV_DIR +ARG GYM_CLI_VENV +ENV UV_INSTALL_DIR=/usr/local/bin UV_PYTHON_PREFERENCE=only-system + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates build-essential && rm -rf /var/lib/apt/lists/* +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +# Upstream Gym main; the finance fork is retired (its fixes landed via PR #2055). +ARG GYM_REPO_URL=https://github.com/NVIDIA-NeMo/Gym.git +ARG GYM_REF=33ef60369f76557e6a6dd828c0bd5f5529624a92 +RUN git clone "${GYM_REPO_URL}" /opt/Gym && cd /opt/Gym && git checkout --detach "${GYM_REF}" && \ + test "$(git rev-parse HEAD)" = "${GYM_REF}" && \ + echo "Gym baked at ${GYM_REF}" + +# CLI venv outside /opt/Gym so a source mount can't shadow it. The `gym` wrapper +# calls the venv python directly: uv's editable entry-point script is unreliable +# under podman/enroot. +RUN uv venv "${GYM_CLI_VENV}" --python "$(command -v python${PYTHON_VERSION})" && \ + cd /opt/Gym && uv pip install --python "${GYM_CLI_VENV}/bin/python" -e "." +RUN printf '#!/bin/sh\nexec "%s/bin/python" -c "from nemo_gym.cli.main import main; main()" "$@"\n' \ + "${GYM_CLI_VENV}" > /usr/local/bin/gym && chmod +x /usr/local/bin/gym + +# Bake per-component venvs. Inline values only satisfy config resolution during +# the dry-run; the recipe sets real values at runtime. +RUN cd /opt/Gym && \ + gym env start --resources-server equivalence_llm_judge --model-type vllm_model \ + +dry_run=true +uv_venv_dir="${GYM_VENV_DIR}" +skip_venv_if_present=false \ + +policy_base_url=http://unset/v1 +policy_api_key=unset +policy_model_name=unset && \ + gym env start --resources-server finance_sec_search --model-type vllm_model \ + +dry_run=true +uv_venv_dir="${GYM_VENV_DIR}" +skip_venv_if_present=false \ + +policy_base_url=http://unset/v1 +policy_api_key=unset +policy_model_name=unset \ + +search_judge_model_base_url=https://api.openai.com/v1 \ + +search_judge_model_api_key=unset +search_judge_model_name=gpt-5-mini +tavily_api_key=null && \ + gym env start --resources-server format_verification/freeform_formatting --model-type vllm_model \ + +dry_run=true +uv_venv_dir="${GYM_VENV_DIR}" +skip_venv_if_present=false \ + +policy_base_url=http://unset/v1 +policy_api_key=unset +policy_model_name=unset + +# Gym calls uvicorn.run(timeout_worker_healthcheck=) (uvicorn>=0.37) but declares +# no floor; assert so a resolver regression fails here, not at the first rollout. +RUN "${GYM_VENV_DIR}/resources_servers/finance_sec_search/.venv/bin/python" -c \ + "import inspect, uvicorn; \ +assert 'timeout_worker_healthcheck' in inspect.signature(uvicorn.run).parameters, uvicorn.__version__; \ +print('uvicorn', uvicorn.__version__, 'OK')" + +# Record the ref before dropping history, so a running container can still report +# which Gym it carries. +RUN echo "${GYM_REF}" > /opt/gym-commit && rm -rf /opt/Gym/.git + +# --- final ------------------------------------------------------------------- +FROM python:${PYTHON_VERSION}-slim +ARG GYM_VENV_DIR +ARG GYM_CLI_VENV +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates procps curl && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/gym-commit /opt/gym-commit +COPY --from=builder /opt/Gym /opt/Gym +COPY --from=builder ${GYM_CLI_VENV} ${GYM_CLI_VENV} +COPY --from=builder ${GYM_VENV_DIR} ${GYM_VENV_DIR} +COPY --from=builder /usr/local/bin/gym /usr/local/bin/gym + +# Let a non-root runtime create new component venvs: open the top + category +# dirs; baked component dirs and venv files stay untouched. +RUN chmod a+rwX ${GYM_VENV_DIR} ${GYM_VENV_DIR}/*/ + +# uv: on-demand venv build for non-baked components (baked ones never use it). +COPY --from=builder /usr/local/bin/uv /usr/local/bin/uv + +# DG-SDG helper deps in the base interpreter for nvflow `python3 -m ...` jobs. +RUN pip install --no-cache-dir orjson pyyaml + +WORKDIR / + +# =========================================================================== +# Security hardening (Trivy/NSPECT scans, 2026-07-08) +# =========================================================================== +# rm all ray_dist.jar copies (shipped 3x): jackson-databind RCE CVE-2026-54512/CVE-2026-54513 (Ray-Java unused); fail build if one survives +RUN find /opt /usr/local -name 'ray_dist.jar' -type f -delete 2>/dev/null; \ + ! find /opt /usr/local -name 'ray_dist.jar' -type f 2>/dev/null | grep -q . diff --git a/dockerfiles/Dockerfile.nemo-rl b/dockerfiles/Dockerfile.nemo-rl index 56c21ad..bfd2293 100644 --- a/dockerfiles/Dockerfile.nemo-rl +++ b/dockerfiles/Dockerfile.nemo-rl @@ -1,131 +1,123 @@ # ============================================================================= -# NVFlow NeMo-RL Container -# ============================================================================= -# Extends the NeMo-RL nightly container with NeMo-Skills and the NeMo-Gym -# finance agent for NVFlow workflows. The base image ships with frozen -# environments and pre-built Ray venvs; this Dockerfile adds the NeMo-Skills -# package, replaces the Gym submodule with a feature branch that includes the -# finance-SEC-search resource server and finance agent, pre-builds all Gym -# component venvs, and relocates paths for Slurm/enroot compatibility. +# NVFlow NeMo-RL trainer (airgapped) for GRPO/SFT `training`. +# +# The base bakes the RL environment but leaves the Gym venvs unbuilt (upstream +# gates that prefetch behind NEMO_GYM_PREFETCH_CONFIGS). This image bakes them, +# so no `uv` resolve happens at job runtime. +# +# Run with `--no-container-mount-home`, which the NeMo-Skills launcher always +# passes: the venvs resolve through /root, which enroot otherwise shadows with +# $HOME. # -# Build: -# docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:latest . +# Build (single-arch, host platform; see docker_instructions.md for multi-arch): +# docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:v0.7.0 . # ============================================================================= - -ARG BASE_IMAGE=nvcr.io/nvidia/nemo-rl:v0.6.0 +ARG BASE_IMAGE=nvcr.io/nvidia/nemo-rl:v0.7.0 FROM ${BASE_IMAGE} -# --- Symlink for NeMo-Skills code that references /opt/NeMo-RL (wrong case) -- -RUN ln -sf /opt/nemo-rl /opt/NeMo-RL - -# --- Upgrade uv (nemo-gym may require newer features than what the nightly ships) -RUN curl -LsSf https://astral.sh/uv/install.sh | sh - -# --- Pre-cache Python interpreter for uv (air-gapped safety net) ------------- -# Gym's cli_setup_command runs `uv venv --python `. If -# skip_venv_if_present ever misses, uv still needs a local interpreter. -RUN /root/.local/bin/uv python install 3.12 - -# --- Install NeMo-Skills into the frozen venv -------------------------------- -ARG NEMO_SKILLS_COMMIT=022904023ad7a83a87662a313cf72e7df5891d55 -RUN git clone https://github.com/NVIDIA-NeMo/Skills.git /opt/NeMo-Skills && \ - cd /opt/NeMo-Skills && git checkout ${NEMO_SKILLS_COMMIT} && \ - /root/.local/bin/uv pip install --python /opt/nemo_rl_venv/bin/python . - -# --- Replace NeMo-Gym submodule with feature branch ------------------------- -# The feature branch includes the finance-SEC-search resource server and -# finance agent that are not yet on main. -ARG NEMO_GYM_BRANCH=ude/finance-sec-search-v2 -RUN rm -rf /opt/nemo-rl/3rdparty/Gym-workspace/Gym && \ - git clone --branch ${NEMO_GYM_BRANCH} \ - https://github.com/NVIDIA-NeMo/Gym.git \ - /opt/nemo-rl/3rdparty/Gym-workspace/Gym +ARG GYM_REF=33ef60369f76557e6a6dd828c0bd5f5529624a92 +ARG NEMO_GYM_CUDA=cu130 +ARG NEMO_GYM_VLLM_VERSION=0.20.0 +ARG TARGETARCH +ARG GYM_SRC=/opt/nemo-rl/3rdparty/Gym-workspace/Gym +ARG GYM_VENV=/opt/ray_venvs/nemo_rl.environments.nemo_gym.NemoGym -# --- Pre-build Gym venv ------------------------------------------------------ -WORKDIR /opt/nemo-rl/3rdparty/Gym-workspace/Gym -RUN /root/.local/bin/uv venv .venv --python 3.12 && \ - . .venv/bin/activate && \ - /root/.local/bin/uv sync --active --extra dev - -# Install finance-specific dependencies into Gym venv -# uvicorn>=0.37.0 is required for timeout_worker_healthcheck support; -# uv sync resolves from the parent nemo-rl workspace lock (0.35.0) instead -# of the Gym lock, so we force the correct version here. -RUN . .venv/bin/activate && \ - /root/.local/bin/uv pip install aiohttp beautifulsoup4 "tavily==1.1.0" tenacity "uvicorn>=0.37.0" - -# --- Symlink component venvs to the main Gym venv --------------------------- -# Each NeMo-Gym component expects its own .venv/; symlinking avoids multi-GB -# duplication and guarantees every component runs with the same packages. -RUN for component in \ - resources_servers/equivalence_llm_judge \ - resources_servers/finance_sec_search \ - responses_api_agents/simple_agent \ - responses_api_agents/finance_agent \ - responses_api_models/openai_model \ - responses_api_models/vllm_model; do \ - dir="/opt/nemo-rl/3rdparty/Gym-workspace/Gym/$component"; \ - [ -d "$dir" ] && ln -sf /opt/nemo-rl/3rdparty/Gym-workspace/Gym/.venv "$dir/.venv"; \ - done - -WORKDIR / - -# --- Install Gym into the NemoGym Ray venv ------------------------------------ -# The pre-built Ray venv from the base image is stale (built from the old Gym -# submodule). Install the new Gym branch editable + all deps so the Ray actor -# can import nemo_gym without missing modules (e.g. gprof2dot, pydot). -RUN /root/.local/bin/uv pip install \ - --python /opt/ray_venvs/nemo_rl.environments.nemo_gym.NemoGym/bin/python \ - -e /opt/nemo-rl/3rdparty/Gym-workspace/Gym - -# --- Align numpy across all Ray venvs to match the main venv ---------------- -# NeMo-Skills may upgrade numpy; mismatched versions cause pickle failures -# when Ray serializes data between the main process and worker processes. -RUN MAIN_NP=$(/opt/nemo_rl_venv/bin/python -c "import numpy; print(numpy.__version__)") && \ - for venv in /opt/ray_venvs/*/; do \ - "$venv/bin/pip" install --no-cache-dir "numpy==$MAIN_NP" 2>/dev/null || true; \ - done - -# --- Relocate /root/.local/ β†’ /opt/ ----------------------------------------- -# enroot/pyxis on Slurm mounts the user's home directory over /root at runtime, -# which shadows everything uv installed there during the Docker build. -# NOTE: Do NOT move /root/.cache/uv β€” base-image venvs symlink into it. -RUN REAL_PYTHON=$(readlink /opt/nemo_rl_venv/bin/python) && \ - mv /root/.local/share/uv/python /opt/uv-python && \ - find /opt/uv-python -maxdepth 1 -type l | while read link; do \ - target=$(readlink "$link") && \ - new_target=$(echo "$target" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \ - ln -sf "$new_target" "$link"; \ - done && \ - NEW_PYTHON=$(echo "$REAL_PYTHON" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \ - ln -sf "$NEW_PYTHON" /opt/nemo_rl_venv/bin/python && \ - sed -i "s|/root/.local/share/uv/python|/opt/uv-python|g" /opt/nemo_rl_venv/pyvenv.cfg && \ - mv /root/.local/bin /opt/uv-bin - -# --- Fix pre-built Ray venvs (same /root/ relocation) ----------------------- -RUN for cfg in /opt/ray_venvs/*/pyvenv.cfg; do \ - sed -i "s|/root/.local/share/uv/python|/opt/uv-python|g" "$cfg"; \ - done && \ - find /opt/ray_venvs/ -type l | while read link; do \ - target=$(readlink "$link") && \ - case "$target" in */root/.local/share/uv/python*) \ - new_target=$(echo "$target" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \ - ln -sf "$new_target" "$link" ;; \ - esac; \ - done - -# --- Fix Gym venv (same /root/ relocation) ---------------------------------- -RUN GYM_VENV=/opt/nemo-rl/3rdparty/Gym-workspace/Gym/.venv && \ - sed -i "s|/root/.local/share/uv/python|/opt/uv-python|g" "$GYM_VENV/pyvenv.cfg" && \ - find "$GYM_VENV" -type l | while read link; do \ - target=$(readlink "$link") && \ - case "$target" in */root/.local/share/uv/python*) \ - new_target=$(echo "$target" | sed "s|/root/.local/share/uv/python|/opt/uv-python|") && \ - ln -sf "$new_target" "$link" ;; \ - esac; \ - done +# scripts/convert_checkpoint_to_hf.sh cd's to /opt/NeMo-RL (wrong case). +RUN ln -sf /opt/nemo-rl /opt/NeMo-RL -# --- Runtime environment ----------------------------------------------------- -ENV VIRTUAL_ENV=/opt/nemo_rl_venv -ENV PATH=/opt/uv-bin:/opt/nemo_rl_venv/bin:$PATH -ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python +# Versions every process joining the Ray cluster must share. Gym's ray floor is +# only >=2.55.1, which would not stop a resolver from moving it. +RUN /opt/nemo_rl_venv/bin/python -c \ + "import numpy, ray; print(f'numpy=={numpy.__version__}'); print(f'ray=={ray.__version__}')" \ + > /opt/nvflow-pins.txt && \ + cat /opt/nvflow-pins.txt + +# Advance in place: generate_fingerprint.py hashes submodule SHAs. +WORKDIR ${GYM_SRC} +RUN git fetch --depth 1 origin ${GYM_REF} && \ + git checkout --detach ${GYM_REF} && \ + test "$(git rev-parse HEAD)" = "${GYM_REF}" + +# Pick up deps Gym declared since the base was built (editable, so source follows). +RUN uv pip install --python ${GYM_VENV}/bin/python \ + --constraint /opt/nvflow-pins.txt -e . && \ + ${GYM_VENV}/bin/python -c "import nemo_gym" + +# Bake one venv per Gym component. Driving the CLI from the actor venv is what +# makes Gym pin each component to that interpreter's ray== and python_version(). +# Gym's vllm==0.20.0 defaults to a cu12 wheel; override to cu130 to match torch. +RUN <<"EOF" bash -eux +case "${TARGETARCH:-amd64}" in arm64) WHEEL_ARCH=aarch64 ;; *) WHEEL_ARCH=x86_64 ;; esac +printf 'vllm @ https://github.com/vllm-project/vllm/releases/download/v%s/vllm-%s-cp38-abi3-manylinux_2_35_%s.whl\n' \ + "${NEMO_GYM_VLLM_VERSION}" "${NEMO_GYM_VLLM_VERSION}" "${WHEEL_ARCH}" > /tmp/gym-vllm.txt +export UV_TORCH_BACKEND="${NEMO_GYM_CUDA}" UV_OVERRIDE=/tmp/gym-vllm.txt UV_LINK_MODE=symlink + +# Empty to start, so skip_venv_if_present can only reuse venvs from this build +# (the shared vllm_model/agent ones), never a stale one. +test -z "$(ls -A "${NEMO_GYM_VENV_DIR}" 2>/dev/null)" + +BAKE="${GYM_VENV}/bin/gym env start --model-type vllm_model +dry_run=true" +BAKE="${BAKE} +uv_venv_dir=${NEMO_GYM_VENV_DIR} +skip_venv_if_present=true" +BAKE="${BAKE} +policy_base_url=http://unset/v1 +policy_api_key=unset +policy_model_name=unset" + +${BAKE} --resources-server equivalence_llm_judge +${BAKE} --resources-server format_verification/freeform_formatting +${BAKE} --resources-server finance_sec_search \ + +search_judge_model_base_url=https://api.openai.com/v1 \ + +search_judge_model_api_key=unset +search_judge_model_name=gpt-5-mini \ + +tavily_api_key=null + +rm -f /tmp/gym-vllm.txt + +# Gym calls uvicorn.run(timeout_worker_healthcheck=) (uvicorn>=0.37) but declares +# no floor; assert so a resolver regression fails here, not at the first rollout. +SEC_VENV="${NEMO_GYM_VENV_DIR}/resources_servers/finance_sec_search/.venv" +test -d "$SEC_VENV" +"$SEC_VENV/bin/python" -c "import inspect, uvicorn; \ +assert 'timeout_worker_healthcheck' in inspect.signature(uvicorn.run).parameters, uvicorn.__version__; \ +print('uvicorn', uvicorn.__version__, 'OK')" +EOF + +WORKDIR /opt/nemo-rl +RUN python tools/generate_fingerprint.py > /opt/nemo_rl_container_fingerprint + +# =========================================================================== +# Security hardening (mirrors Dockerfile.vllm) +# =========================================================================== +# Keep headers installed: triton and TransformerEngine JIT-compile at run time. +RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* + +# ray_dist.jar: jackson-databind RCE CVE-2026-54512/CVE-2026-54513, Ray-Java +# unused. Most copies are venv symlinks into the uv cache, so the cache holds the +# only real file and must be searched -- but never delete the cache itself, which +# the worker venvs symlink into for everything else. +RUN find /usr/local /opt /root/.cache/uv -name 'ray_dist.jar' -delete 2>/dev/null; \ + ! find /usr/local /opt /root/.cache/uv -name 'ray_dist.jar' 2>/dev/null | grep -q . && \ + /opt/nemo_rl_venv/bin/python -c "import ray; print('ray OK', ray.__version__)" + +# Ray refuses to join a cluster on a different Ray or Python version. numpy is +# gated only in the actor venvs, which exchange pickled arrays; Gym components +# talk HTTP/orjson, so a difference there is reported, not fatal. +RUN <<"EOF" bash -eux +ref() { /opt/nemo_rl_venv/bin/python -c "import $1 as m, platform; print(m.__version__)"; } +REF_PY=$(/opt/nemo_rl_venv/bin/python -c "import platform; print(platform.python_version())") +REF_RAY=$(ref ray); REF_NP=$(ref numpy) +FOUND=0 +for py in /opt/ray_venvs/*/bin/python "${NEMO_GYM_VENV_DIR}"/*/*/.venv/bin/python; do + [ -x "$py" ] || continue + FOUND=$((FOUND + 1)) + got() { "$py" -c "import $1 as m; print(m.__version__)" 2>/dev/null || true; } + V=$("$py" -c "import platform; print(platform.python_version())") + [ "$V" = "$REF_PY" ] || { echo "python skew: $py is $V, want $REF_PY"; exit 1; } + RAY=$(got ray) + [ -z "$RAY" ] || [ "$RAY" = "$REF_RAY" ] || { echo "ray skew: $py has $RAY, want $REF_RAY"; exit 1; } + NP=$(got numpy) + case "$py" in + /opt/ray_venvs/*) [ -z "$NP" ] || [ "$NP" = "$REF_NP" ] || \ + { echo "numpy skew: $py has $NP, want $REF_NP"; exit 1; } ;; + *) [ -z "$NP" ] || [ "$NP" = "$REF_NP" ] || echo "note: gym venv $py numpy $NP vs $REF_NP" ;; + esac +done +[ "$FOUND" -gt 0 ] || { echo "no venvs inspected"; exit 1; } +echo "python ${REF_PY} / ray ${REF_RAY} consistent across ${FOUND} venvs; numpy ${REF_NP} in actor venvs" +EOF diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills index a67c6eb..e390c3a 100644 --- a/dockerfiles/Dockerfile.nemo-skills +++ b/dockerfiles/Dockerfile.nemo-skills @@ -1,49 +1,75 @@ # ============================================================================= # NVFlow NeMo-Skills Container -# ============================================================================= -# Self-contained Dockerfile that builds the NeMo-Skills evaluation container -# with all required packages pre-installed. # -# Build: -# docker build -f dockerfiles/Dockerfile.nemo-skills -t nvflow-nemo-skills:latest . +# 1. UPSTREAM (NeMo-Skills' Dockerfile at a pinned commit, kept diffable), +# 2. NVFLOW (packages the workflow steps import), 3. CVE (scan-driven only -- +# delete it for a stock image). Upstream: +# https://github.com/NVIDIA-NeMo/Skills/blob/main/dockerfiles/Dockerfile.nemo-skills # -# Upstream source: -# https://github.com/NVIDIA-NeMo/Skills/blob/main/dockerfiles/Dockerfile.nemo-skills +# Build (single-arch, host platform; see docker_instructions.md for multi-arch): +# docker build --no-cache \ +# -f dockerfiles/Dockerfile.nemo-skills -t nvflow-nemo-skills:v1.1.2 . # ============================================================================= +# Section 3 item; Docker needs builder stages first, so delete it with the COPY in +# section 3. wandb 0.28.1 (newest release) bundles wandb-core built with Go 1.26.4, +# grpc-go v1.82.0 and x/text v0.38.0; commit e118409 is the unreleased fix. +# Cross-compiled from BUILDPLATFORM to skip arm64 emulation. +ARG WANDB_CORE_COMMIT=e1184091520c9b44aa1096fdb27b2f4bf52f26d7 +FROM --platform=$BUILDPLATFORM golang:1.26.5 AS wandb-core-builder +ARG WANDB_CORE_COMMIT +ARG TARGETARCH +RUN git init /src/wandb && cd /src/wandb && \ + git remote add origin https://github.com/wandb/wandb.git && \ + git sparse-checkout init --cone && git sparse-checkout set core && \ + git fetch --depth 1 origin "${WANDB_CORE_COMMIT}" && git checkout --detach FETCH_HEAD +RUN cd /src/wandb/core && \ + CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ + -tags "disable_grpc_modules parquet_read_only" \ + -ldflags "-s -w -X main.commit=${WANDB_CORE_COMMIT}" -mod=vendor \ + -o /wandb-core ./cmd/wandb-core && \ + go version -m /wandb-core | grep -F "go1.26.5" && \ + go version -m /wandb-core | grep -E "google\.golang\.org/grpc[[:space:]]+v1\.82\.1([[:space:]]|$)" && \ + go version -m /wandb-core | grep -E "golang\.org/x/text[[:space:]]+v0\.40\.0([[:space:]]|$)" -# Clone NeMo-Skills at a pinned commit (replaces build-context COPY commands) +# NeMo-Skills at a pinned commit, replacing upstream's build-context COPYs. FROM scratch AS nemo-skills-src -ARG NEMO_SKILLS_COMMIT=022904023ad7a83a87662a313cf72e7df5891d55 +ARG NEMO_SKILLS_COMMIT=e06c9b900177be3f60d6a3f99135bb5de9af9bed ADD --keep-git-dir=true https://github.com/NVIDIA-NeMo/Skills.git#${NEMO_SKILLS_COMMIT} / # =========================================================================== -# BEGIN UPSTREAM (adapted from NeMo-Skills Dockerfile.nemo-skills) -# Source: https://github.com/NVIDIA-NeMo/Skills/blob/0229040/dockerfiles/Dockerfile.nemo-skills -# Modifications: -# - COPY commands changed to COPY --from=nemo-skills-src -# - Added `tzdata` to apt packages (required by pyarrow/pandas; populates -# /usr/share/zoneinfo so libc tz lookups resolve, e.g. "UTC") +# 1. BEGIN UPSTREAM -- github.com/NVIDIA-NeMo/Skills @ e06c9b90 +# Deviations, so a diff after re-syncing shows only these: +# - COPY -> COPY --from=nemo-skills-src (pinned SHA instead of build context) +# - google-research not cloned (unused here; carries Criticals) +# - `cd /tmp` before the nltk download (upstream defect; see below) +# - ARG WANDB_CORE_COMMIT redeclared, for the CVE section's COPY +# Do not add --no-install-recommends below: it drops gpg-agent, which +# add-apt-repository needs for the apptainer PPA key. # =========================================================================== +# using ubuntu instead of debian for easier apptainer installation on arm64 FROM ubuntu:22.04 +ARG WANDB_CORE_COMMIT +# Install Python and other dependencies RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ + apt-get install -y \ python3.10 \ python3-pip \ curl \ wget \ git \ git-lfs \ - ffmpeg \ - tzdata && \ + ffmpeg && \ ln -s /usr/bin/python3 /usr/bin/python && \ rm -rf /var/cache/apt/archives /var/lib/apt/lists/* -RUN pip install --upgrade pip setuptools uv +RUN pip install --upgrade pip setuptools "uv>=0.11.10" +# Update package lists and install apptainer for arm64 +# https://apptainer.org/docs/admin/1.1/installation.html RUN apt update && \ apt install -y software-properties-common && \ add-apt-repository -y ppa:apptainer/ppa && \ @@ -52,8 +78,18 @@ RUN apt update && \ apt update && apt install -y apptainer-suid && \ rm -rf /var/cache/apt/archives /var/lib/apt/lists/* +# Apply security patches for PackageKit, pulled in transitively by software-properties-common. +# Ubuntu 22.04 has published 1.2.5-2ubuntu3.1 with the fix for the local privilege escalation CVE. +RUN apt-get update && \ + apt-get install --only-upgrade -y \ + packagekit \ + packagekit-tools \ + libpackagekit-glib2-18 \ + gir1.2-packagekitglib-1.0 && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists/* + +# for ifeval benchmark -- google-research clone skipped (see deviations) RUN mkdir /opt/benchmarks -RUN git clone https://github.com/google-research/google-research.git /opt/benchmarks/google-research --depth=1 RUN git clone https://github.com/ShishirPatil/gorilla.git /opt/gorilla RUN cd /opt/gorilla && git checkout 86d0374d0db52623c5092a73f82c22b87b7e9a25 @@ -61,6 +97,7 @@ RUN cd /opt/gorilla/berkeley-function-call-leaderboard && pip install --no-cache RUN apt remove -y python3-blinker +# ifbench ARG IFBENCH_COMMIT=c6767a19bd82ac0536cab950f2f8f6bcc6fabe7c ARG IFBENCH_REPO=https://github.com/allenai/IFBench.git ARG IFBENCH_DIR=/opt/benchmarks/IFBench @@ -68,24 +105,32 @@ RUN git init "$IFBENCH_DIR" && cd "$IFBENCH_DIR" && git remote add origin "$IFBE git fetch --depth 1 origin "${IFBENCH_COMMIT}" && git reset --hard FETCH_HEAD RUN cd ${IFBENCH_DIR} && pip install -r requirements.txt +# removing on-the-fly installation in ifbench to avoid conflicts from parallel jobs COPY --from=nemo-skills-src /dockerfiles/ifbench.patch /opt/benchmarks/IFBench/ifbench.patch RUN cd /opt/benchmarks/IFBench && git apply ifbench.patch +# nltk >=3.10.1 blocks imports resolving inside the CWD; at CWD=/ that matches +# every stdlib path, so `import nltk` needs the `cd`. Downloads use fixed paths. RUN pip install langdetect absl-py immutabledict nltk ipython && \ - python -c "import nltk; from spacy.cli import download; nltk.download('punkt'); nltk.download('punkt_tab'); \ + cd /tmp && python -c "import nltk; from spacy.cli import download; nltk.download('punkt'); nltk.download('punkt_tab'); \ nltk.download('stopwords'); nltk.download('averaged_perceptron_tagger_eng'); download('en_core_web_sm')" +# we aren't copying main nemo_skills folder as it will always be mounted from host +# but we do want to install all requirements in the container directly RUN mkdir -p /opt/NeMo-Skills/requirements /opt/NeMo-Skills/core COPY --from=nemo-skills-src /pyproject.toml /opt/NeMo-Skills/pyproject.toml COPY --from=nemo-skills-src /README.md /opt/NeMo-Skills/README.md COPY --from=nemo-skills-src /requirements/ /opt/NeMo-Skills/requirements/ COPY --from=nemo-skills-src /core/requirements.txt /opt/NeMo-Skills/core/requirements.txt +# installing sdp in container only RUN pip install git+https://github.com/NVIDIA/NeMo-speech-data-processor@29b9b1ec0ceaf3ffa441c1d01297371b3f8e11d2 ARG CACHEBUST=4 -RUN echo "httpx>=0.28.1" > /tmp/overrides.txt && \ - uv pip install --system --no-cache --override /tmp/overrides.txt \ - -r /opt/NeMo-Skills/core/requirements.txt \ - -r /opt/NeMo-Skills/requirements/pipeline.txt +# Install via `uv pip` from the project directory so [tool.uv].override-dependencies +# in pyproject.toml (which relaxes leptonai's httpx==0.27.2 pin so litellm 1.83.x +# can be installed) is picked up. Plain pip ignores [tool.uv] and the resolver fails. +RUN cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \ + -r core/requirements.txt -r requirements/pipeline.txt +# Fix http mismatch between lepton and dggs by manually downloading dggs here RUN pip install ddgs # =========================================================================== @@ -94,12 +139,14 @@ RUN pip install ddgs # =========================================================================== -# NVFlow Additional Packages -# =========================================================================== -# Pre-install packages used by NVFlow workflow steps so they are available -# at runtime without needing to download anything. +# 2. NVFLOW -- imported by workflow steps; baked so no job downloads at launch. # =========================================================================== +# tzdata populates /usr/share/zoneinfo for pyarrow/pandas libc tz lookups. +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \ + rm -rf /var/cache/apt/archives /var/lib/apt/lists/* + RUN pip install --no-cache-dir --ignore-requires-python \ jsonlines \ tiktoken \ @@ -110,15 +157,12 @@ RUN pip install --no-cache-dir --ignore-requires-python \ "model-library==0.1.8" \ "compute-eval @ git+https://github.com/NVIDIA/compute-eval.git@2d14770" -# Pre-cache tiktoken encodings so no downloads are needed at runtime. # cl100k_base is used by question_context_utils.py for token counting. ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache RUN mkdir -p /opt/tiktoken_cache && \ python3 -c "import tiktoken; tiktoken.get_encoding('cl100k_base')" -# SEC data-prep dependencies for workflow-2 (download_sec_filings) and -# workflow-3 step-0 (create_seed_data). Baked in so the recipe-level -# installation_command can stay empty -- no PyPI fetch at job-launch time. +# SEC data prep: workflow-2 download_sec_filings, workflow-3 create_seed_data. RUN pip install --no-cache-dir --ignore-requires-python \ edgartools==5.20.2 \ sec-parser \ @@ -130,9 +174,98 @@ RUN pip install --no-cache-dir --ignore-requires-python \ requests \ beautifulsoup4 -# Fail-fast smoke check so the image won't ship missing a needed dep. RUN python -c "from edgar import set_identity; \ import sec_parser, pandas, pyarrow, httpx, tzdata, requests; \ - from bs4 import BeautifulSoup; \ - from datasets import load_dataset; \ - print('SEC-prep + create_seed_data deps OK')" + from bs4 import BeautifulSoup; from datasets import load_dataset; \ + print('NVFlow deps OK')" + + +# =========================================================================== +# 3. CVE -- scan findings only, not functionality; drop each once upstream ships +# the fix. Floors upstream already satisfies are asserted, never re-pinned -- +# re-pinning core packages is what silently broke tokenizers once. +# =========================================================================== + +# libssl3 -> High CVE-2026-45447. linux-libc-dev's ~176 header findings are inert +# but fix-available; nothing compiles at runtime. +RUN apt-get update && apt-get install -y --only-upgrade --no-install-recommends libssl3 && \ + apt-get purge -y linux-libc-dev && apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +# cssutils imports more_itertools at module level, and pip earlier resolved that +# against an apt copy which autoremove above then took away. Own it with pip. +RUN pip install --no-cache-dir --ignore-installed more-itertools + +# starlette and lxml have no satisfiable version: the fixes (GHSA-86qp-5c8j-p5mr, +# -x746-7m8f-x49c, -wqp7-x3pw-xc5r, -jp82-jpqv-5vv3, -82w8-qh3p-5jfq; and +# GHSA-vfmq-68hx-4jfw) landed in 1.x and 6.1.0, above the caps in leptonai's +# instrumentator pin and in sec-parser. Forced, and allowlisted in the pip check. +# GitPython and datamodel-code-generator are not in upstream's graph, so they are +# requested here. Full requirement set, no --upgrade: no ceiling missed, nothing +# else moves. +RUN printf '%s\n' 'starlette>=1.3.1' 'lxml>=6.1.0' > /tmp/cve-overrides.txt && \ + cd /opt/NeMo-Skills && uv pip install --system --no-cache-dir \ + --override /tmp/cve-overrides.txt \ + -r core/requirements.txt -r requirements/pipeline.txt \ + 'GitPython>=3.1.55' 'datamodel-code-generator>=0.64.0' + +# Pairs with the wandb-core-builder stage; the commit is injected at link time. +COPY --from=wandb-core-builder /wandb-core /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core +RUN /usr/local/lib/python3.10/dist-packages/wandb/bin/wandb-core --help 2>&1 | \ + grep -F "Commit SHA: ${WANDB_CORE_COMMIT}" + +# Benchmark trees finance never runs, and Ray's Java jar (jackson-databind +# findings; Ray is used only through Python). bfcl_eval was installed editable, so +# uninstall it rather than leave a dangling dist-info. +RUN uv pip uninstall --system bfcl_eval || true && \ + rm -rf /opt/gorilla /opt/benchmarks/IFBench /usr/local/lib/python3*/dist-packages/ray/jars && \ + ! python -c "import bfcl_eval" 2>/dev/null && \ + [ -z "$(find /usr/local/lib/python3*/dist-packages -maxdepth 1 -name '*bfcl*')" ] && \ + ! find /usr/local /opt -name 'ray_dist.jar' -type f 2>/dev/null | grep -q . && \ + python -c "import ray; print('bfcl/IFBench removed, ray OK:', ray.__version__)" + +# Asserted, not pinned, so a regression fails the build rather than being papered +# over. wandb is exact: the patched core binary must match its Python protocol. +RUN python -c "from importlib.metadata import version as v; from packaging.requirements import Requirement as R; \ + floors = ['starlette>=1.3.1', 'GitPython>=3.1.55', 'datamodel-code-generator>=0.64.0', \ + 'nltk>=3.10.0', 'wandb==0.28.1', 'litellm>=1.84.10', 'lxml>=6.1.0', \ + 'httpx>=0.28.1', 'urllib3>=2.6.3', 'msgpack>=1.2.1', 'setuptools>=78.1.1', \ + 'click>=8.2,<9', 'typer>=0.16,<0.27', 'mcp<2.0']; \ + bad = ['%s %s needs %s' % (r.name, v(r.name), r.specifier) for r in map(R, floors) \ + if not r.specifier.contains(v(r.name), prereleases=True)]; \ + assert not bad, bad; print('CVE floors OK')" + +# Gate what the upgrades put at risk, here rather than on the cluster: transformers +# enforces its tokenizers range at import, and lxml is forced past sec-parser's +# cap, so parse a document rather than only importing it. +RUN python -c "from transformers import AutoTokenizer; import transformers, tokenizers, lxml.etree, sec_parser; \ + import cssutils; from litellm import completion; from fastapi import FastAPI; FastAPI(); \ + els = sec_parser.Edgar10QParser().parse('

Item 2. Management Discussion

Revenue rose 12 percent.

'); \ + assert len(els) >= 2, els; \ + print('gates OK: transformers', transformers.__version__, 'tokenizers', tokenizers.__version__, \ + 'lxml', lxml.etree.__version__, '/', len(els), 'sec elements')" + +RUN WANDB_MODE=offline WANDB_SILENT=true WANDB_DIR=/tmp/wandb-smoke \ + python -c "import wandb; run = wandb.init(project='nvflow-security-smoke', name='offline'); \ + run.log({'metric': 1.0}); run.finish(); print('wandb offline smoke OK')" && \ + rm -rf /tmp/wandb-smoke + +# Whole-environment check, last so nothing can invalidate it. Matched on the exact +# pair, so a new break fails even in a listed package. The first three are +# upstream's own (its pyproject overrides, and sdp's numpy pin). +RUN printf '%s\n' \ + 'leptonai .* requirement httpx' \ + 'torchx .* requirement urllib3' \ + 'sdp .* requirement numpy' \ + 'prometheus-fastapi-instrumentator .* requirement starlette' \ + 'sec-parser .* requirement lxml' \ + 'No broken requirements found' \ + > /tmp/pipcheck-allow.txt; \ + pip check > /tmp/pipcheck.txt 2>&1 || true; \ + if grep -vEf /tmp/pipcheck-allow.txt /tmp/pipcheck.txt | grep -q '[^[:space:]]'; then \ + echo "ERROR: unexpected dependency inconsistency:"; cat /tmp/pipcheck.txt; exit 1; \ + fi; \ + cat /tmp/pipcheck.txt + +# uv's sdist cache leaves Git metadata that nSpect's global policy flags. +RUN rm -rf /root/.cache/uv /tmp/cve-overrides.txt /tmp/pipcheck.txt /tmp/pipcheck-allow.txt diff --git a/dockerfiles/Dockerfile.nvflow b/dockerfiles/Dockerfile.nvflow new file mode 100644 index 0000000..7da7d5a --- /dev/null +++ b/dockerfiles/Dockerfile.nvflow @@ -0,0 +1,151 @@ +# syntax=docker/dockerfile:1 +# ============================================================================= +# NVFlow client (launcher) image -- the `nflow` CLI + baked venv for airgap use. +# +# Lean, unprivileged orchestration image: `nflow` submits Slurm jobs over an SSH +# tunnel, so no Slurm client / munge / podman / enroot is needed inside. Worker +# images stay as .sqsh on the cluster (referenced in my_cluster.yaml), not here. +# +# Requires a source with the tunnel-aware launcher (feature/finance-rl-grpo or +# later); the v1.1.1 release cannot launch off-cluster over ssh_tunnel. +# +# Build from the repo root, on a committed tree (.baked_commit records HEAD). +# .dockerignore drops cache/, .venv and secrets; .git is kept for hatch-vcs +# versioning, then squashed to a single history-free commit so the image ships +# no repo history. Single-arch, host platform; see docker_instructions.md for +# multi-arch: +# docker build -f dockerfiles/Dockerfile.nvflow -t nvflow-client:v1.1.2 . +# +# Usage: see docs/remote-launch.md. +# ============================================================================= + +ARG PYTHON_VERSION=3.12 +# Pin the multi-arch Ubuntu 24.04 manifest so a rebuild cannot silently move to +# a new OS release. The previous floating python:3.12-slim tag moved to Debian +# 13 and introduced 17 CRITICAL + 40 HIGH OS-package findings per architecture. +ARG UBUNTU_IMAGE=ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 + +# --- builder: resolve the venv (build tools stay out of the final image) ----- +FROM ${UBUNTU_IMAGE} AS builder +ARG PYTHON_VERSION +ENV DEBIAN_FRONTEND=noninteractive \ + UV_INSTALL_DIR=/usr/local/bin \ + UV_CACHE_DIR=/opt/uv-cache \ + UV_PYTHON_PREFERENCE=only-system + +RUN apt-get update && apt-get install -y --no-install-recommends \ + "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-venv" \ + git curl ca-certificates build-essential \ + && rm -rf /var/lib/apt/lists/* + +# uv on a system path (not /root/.local) so it survives enroot's $HOME remap. +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +COPY . /opt/nvflow +WORKDIR /opt/nvflow +# Record the baked commit for provenance, and hide it from git so the baked tree +# stays clean (nemo-run packages via `git archive` of HEAD regardless). +RUN git rev-parse HEAD > /opt/nvflow/.baked_commit 2>/dev/null || echo unknown > /opt/nvflow/.baked_commit && \ + echo '.baked_commit' >> /opt/nvflow/.git/info/exclude + +# --frozen pins to the committed uv.lock; --no-dev skips the dev group (debugger). +RUN uv venv .venv --python "$(command -v python${PYTHON_VERSION})" && \ + uv sync --frozen --no-dev +# Strip the vendored wandb "core" Go binary. wandb arrives transitively via +# nemo-skills, but the launcher never calls wandb.init()/sync (only the Python +# API is imported), so wandb-core is dead weight that vendors Go stdlib/grpc/ +# x-crypto CVEs (e.g. CVE-2025-68121, CVE-2026-33186) and dominates image scans. +# Deleting the binary keeps the wandb Python package importable for nemo-skills. +# Strip Ray's bundled Java jar (site-packages/ray/jars/ray_dist.jar). Ray arrives +# transitively via nemo-skills; the launcher drives Ray purely through the Python +# Ray Jobs API, so the Java jar is dead weight carrying the jackson-databind HIGH +# CVEs (CVE-2026-54512 / CVE-2026-54513) plus other vendored Java libs +# (guava/gson/jaxb). Removing the whole jars/ dir clears them without affecting Ray. +RUN rm -rf /opt/uv-cache && \ + rm -f /opt/nvflow/.venv/lib/python*/site-packages/wandb/bin/wandb-core && \ + rm -rf /opt/nvflow/.venv/lib/python*/site-packages/ray/jars && \ + find /opt/nvflow -depth -type d -name __pycache__ -exec rm -rf {} + && \ + find /opt/nvflow -type f -name '*.pyc' -delete +# Fail-fast: Ray must still import after the jar strip (Python Ray Jobs API intact). +RUN /opt/nvflow/.venv/bin/python -c "import ray; print('ray OK after jar strip:', ray.__version__)" + +# Ship a history-free repo. hatch-vcs versioning already ran during `uv sync` +# (static nvflow/_version.py written above), and nemo-run only needs +# `git archive HEAD` (current tree) -- a distributed image must not carry repo +# history. Replace the full .git with a single snapshot commit that reproduces +# the EXACT original tracked set: capture `git ls-files` first, then re-add it +# with --force so tracked-but-gitignored files (e.g. recipes/finance/data/ +# __init__.py, which drives stage auto-discovery) are preserved. Plain +# `git add -A` respects .gitignore and would silently drop them, changing what +# `git archive HEAD` ships to the compute nodes. The assertion guards this. +RUN cd /opt/nvflow && \ + git ls-files -z > /tmp/tracked && \ + rm -rf .git && \ + git init -q -b main && \ + git -c user.email=release@nvidia.com -c user.name=nvflow add --pathspec-from-file=/tmp/tracked --pathspec-file-nul --force && \ + git -c user.email=release@nvidia.com -c user.name=nvflow commit -q -m "nvflow baked release snapshot" && \ + git ls-files -z | sort -z > /tmp/after && sort -z /tmp/tracked > /tmp/before && \ + cmp -s /tmp/before /tmp/after && echo "snapshot tree == original tracked set" && \ + rm -f /tmp/tracked /tmp/before /tmp/after + +# --- final: slim runtime = base + venv + source(.git snapshot) + launcher ------ +FROM ${UBUNTU_IMAGE} +ARG PYTHON_VERSION +ARG CA_CERTIFICATES_VERSION=20260601~24.04.1 +ARG GIT_VERSION=1:2.43.0-1ubuntu7.3 +ARG OPENSSH_CLIENT_VERSION=1:9.6p1-3ubuntu13.18 +ARG PYTHON_DEB_VERSION=3.12.3-1ubuntu0.15 +ARG RSYNC_VERSION=3.2.7-1ubuntu1.5 +LABEL org.opencontainers.image.title="nvflow-client" \ + org.opencontainers.image.description="NVFlow launcher (nflow CLI) for airgap use" + +# Launcher runtime deps: git (nemo-run `git archive`), openssh-client (ssh_tunnel), +# rsync (code sync to job_dir), ca-certificates, and the interpreter backing the +# copied venv. Pin the security-updated Ubuntu packages: if an exact version +# leaves the archive, fail the build for an intentional refresh instead of +# silently accepting a vulnerable package set. +RUN apt-get update && apt-get install -y --no-install-recommends \ + "python${PYTHON_VERSION}=${PYTHON_DEB_VERSION}" \ + "git=${GIT_VERSION}" \ + "openssh-client=${OPENSSH_CLIENT_VERSION}" \ + "rsync=${RSYNC_VERSION}" \ + "ca-certificates=${CA_CERTIFICATES_VERSION}" \ + && rm -rf /var/lib/apt/lists/* + +# ssh_tunnel host-key handling. nemo-run authenticates the tunnel via paramiko +# (given the key explicitly), but then rsyncs code with plain `ssh -i `, +# which reads known_hosts from $HOME/.ssh β€” NOT from the mounted /opt/ssh. On a +# fresh container $HOME/.ssh is empty, so rsync dies with "Host key verification +# failed". Point ssh at the *mounted* known_hosts and auto-accept a first-ever +# connect (written back to the mounted dir, so it persists) β€” no manual priming. +# System-wide via the ssh_config Include, so it holds for any HOME/uid, enroot or +# docker, on-cluster or off. Deliberately no IdentityFile here: nemo-run passes +# -i explicitly, and this stays agnostic to the user's key name. +RUN printf 'Host *\n UserKnownHostsFile /opt/ssh/known_hosts\n StrictHostKeyChecking accept-new\n' \ + > /etc/ssh/ssh_config.d/10-nvflow-tunnel.conf + +COPY --from=builder /usr/local/bin/uv /usr/local/bin/uv +COPY --from=builder /opt/nvflow /opt/nvflow + +# System gitconfig (not /root, which enroot remaps) so `git archive` doesn't trip +# git's "dubious ownership" guard when the container runs as a mapped uid. +RUN git config --system --add safe.directory /opt/nvflow + +# enroot ignores ENV PATH; symlink the nflow console script onto the default PATH. +RUN ln -sf /opt/nvflow/.venv/bin/nflow /usr/local/bin/nflow + +# UV_OFFLINE + UV_NO_SYNC: `uv run nflow` runs in the baked venv with no network +# and no pre-run sync (a sync would try to fetch the skipped dev group and fail). +# NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK: the image bakes a fixed committed +# snapshot; nemo-run packages HEAD via `git archive`, so its uncommitted-changes +# gate is a false positive here (build artifacts like the venv make the tree look +# dirty). Disabling it lets the launcher package the baked commit unattended. +ENV VIRTUAL_ENV=/opt/nvflow/.venv \ + PATH=/opt/nvflow/.venv/bin:/usr/local/bin:$PATH \ + NEMO_SKILLS_CONFIG_DIR=/opt/nvflow/cluster_configs \ + UV_CACHE_DIR=/tmp/uv-cache \ + UV_OFFLINE=1 \ + UV_NO_SYNC=1 \ + NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK=1 + +WORKDIR /opt/nvflow diff --git a/dockerfiles/Dockerfile.vllm b/dockerfiles/Dockerfile.vllm index 46bb563..8135f10 100644 --- a/dockerfiles/Dockerfile.vllm +++ b/dockerfiles/Dockerfile.vllm @@ -1,27 +1,41 @@ # ============================================================================= # NVFlow vLLM Container # ============================================================================= -# Self-contained Dockerfile that builds the vLLM inference container with -# pre-cached tokenizer encodings. +# vLLM inference container: adds SAM 3.1 and pre-caches tokenizer encodings so +# it serves airgapped. VLLM_VERSION selects the base tag -- both vLLM images +# build from this file and ship in the nvflow-vllm repo under different tags. # -# Build: -# docker build -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:latest . +# Builds below are single-arch (host platform); see docker_instructions.md for +# multi-arch. +# +# Build (SDG / eval): +# docker build -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:v0.22.0 . +# +# Build (GRPO rollouts + judge; matches NeMo-RL v0.7.0's colocated vLLM): +# docker build --build-arg VLLM_VERSION=v0.20.0 \ +# -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:v0.20.0 . # # Upstream source: # https://github.com/NVIDIA-NeMo/Skills/blob/main/dockerfiles/Dockerfile.vllm # ============================================================================= +ARG VLLM_VERSION=v0.22.0 + +FROM scratch AS sam3-src +ARG SAM3_COMMIT=a51b9f498c84824a94702cc289ed75d9cc544c64 +ADD --keep-git-dir=true https://github.com/facebookresearch/sam3.git#${SAM3_COMMIT} / + # =========================================================================== # BEGIN UPSTREAM (NeMo-Skills Dockerfile.vllm) # =========================================================================== -ARG VLLM_VERSION=v0.18.1 FROM vllm/vllm-openai:${VLLM_VERSION} +RUN pip install ray RUN pip install "vllm[audio]" +# Required by vLLM for Qwen-VL model family (runtime dependency, not directly imported) RUN pip install qwen-vl-utils -RUN pip install ray # =========================================================================== # END UPSTREAM @@ -31,19 +45,36 @@ RUN pip install ray # =========================================================================== # NVFlow Additional Layers # =========================================================================== -# Pre-cache tokenizer encodings so no downloads are needed at runtime. +# Add SAM 3.1 without disturbing vLLM's tested Torch/NumPy stack: install only +# its missing deps + source with --no-deps (SAM's numpy<2 pin would downgrade). # =========================================================================== +COPY --from=sam3-src / /opt/sam3 +RUN pip install --no-cache-dir --no-deps \ + ftfy==6.1.1 \ + iopath==0.1.10 \ + portalocker==3.2.0 \ + pycocotools==2.0.11 \ + wcwidth==0.2.14 && \ + pip install --no-cache-dir --no-deps --no-build-isolation -e /opt/sam3 && \ + rm -rf /opt/sam3/.git + +# `vllm[audio]` and `ray` above are unpinned; assert the base tag's vLLM version +# survived. Ray is only reported -- these servers run standalone. +ARG VLLM_VERSION +RUN python3 -c "import numpy, sam3, torch, vllm, ray; \ +from sam3.model_builder import build_sam3_image_model; \ +assert vllm.__version__.startswith('${VLLM_VERSION#v}'), f'vllm {vllm.__version__} != ${VLLM_VERSION#v}'; \ +print(f'SAM 3.1 + vLLM imports OK: numpy={numpy.__version__}, torch={torch.__version__}, vllm={vllm.__version__}, ray={ray.__version__}')" + +# Pre-cache tokenizer encodings so no downloads are needed at runtime. ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache ENV TIKTOKEN_RS_CACHE_DIR=/opt/tiktoken_cache ENV TIKTOKEN_ENCODINGS_BASE=/opt/tiktoken_cache RUN mkdir -p /opt/tiktoken_cache -# Download tiktoken encoding files explicitly with curl. -# The Rust tiktoken-rs client inside openai_harmony fails to download under -# QEMU arm64 emulation (docker buildx), so we fetch them reliably here and -# point TIKTOKEN_ENCODINGS_BASE at the directory. This also makes the image -# fully air-gapped on both amd64 and arm64. +# Fetch tiktoken encodings explicitly -- the Rust tiktoken-rs client in +# openai_harmony fails to download under QEMU arm64. Keeps the image airgapped. RUN curl -fSL -o /opt/tiktoken_cache/o200k_base.tiktoken \ https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken && \ curl -fSL -o /opt/tiktoken_cache/cl100k_base.tiktoken \ @@ -54,3 +85,13 @@ RUN python3 -c "\ from openai_harmony import load_harmony_encoding, HarmonyEncodingName; \ load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS); \ print('openai_harmony encoding loaded OK')" + +# =========================================================================== +# Security hardening (Trivy/NSPECT wave scans, 2026-07-07) +# =========================================================================== +# apt upgrade for base-channel security fixes (linux-libc-dev/gnupg/openssl = 227 of 233 HIGH/CRIT Trivy); headers stay INSTALLED for vLLM triton JIT +RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* + +# rm ray_dist.jar: jackson-databind RCE CVE-2026-54512/CVE-2026-54513 (Ray-Java unused); fail build if one survives +RUN find /usr/local /opt -name 'ray_dist.jar' -type f -delete 2>/dev/null; \ + ! find /usr/local /opt -name 'ray_dist.jar' -type f 2>/dev/null | grep -q . diff --git a/dockerfiles/Dockerfile.vllm-grpo b/dockerfiles/Dockerfile.vllm-grpo deleted file mode 100644 index 3ade4f7..0000000 --- a/dockerfiles/Dockerfile.vllm-grpo +++ /dev/null @@ -1,55 +0,0 @@ -# ============================================================================= -# NVFlow vLLM-GRPO Container -# ============================================================================= -# vLLM container pinned to v0.17.1 for GRPO rollouts and judge inference. -# The main vLLM container (Dockerfile.vllm) uses v0.18.1 for SDG/eval. -# -# Build: -# docker build -f dockerfiles/Dockerfile.vllm-grpo -t nvflow-vllm-grpo:latest . -# -# Upstream source: -# https://github.com/NVIDIA-NeMo/Skills/blob/main/dockerfiles/Dockerfile.vllm -# ============================================================================= - - -# =========================================================================== -# BEGIN UPSTREAM (NeMo-Skills Dockerfile.vllm) -# =========================================================================== - -ARG VLLM_VERSION=v0.17.1 -FROM vllm/vllm-openai:${VLLM_VERSION} - -RUN pip install "vllm[audio]" -RUN pip install qwen-vl-utils - -# =========================================================================== -# END UPSTREAM -# =========================================================================== - - -# =========================================================================== -# NVFlow Additional Layers -# =========================================================================== -# Pre-cache tokenizer encodings so no downloads are needed at runtime. -# =========================================================================== - -ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache -ENV TIKTOKEN_RS_CACHE_DIR=/opt/tiktoken_cache -ENV TIKTOKEN_ENCODINGS_BASE=/opt/tiktoken_cache -RUN mkdir -p /opt/tiktoken_cache - -# Download tiktoken encoding files explicitly with curl. -# The Rust tiktoken-rs client inside openai_harmony fails to download under -# QEMU arm64 emulation (docker buildx), so we fetch them reliably here and -# point TIKTOKEN_ENCODINGS_BASE at the directory. This also makes the image -# fully air-gapped on both amd64 and arm64. -RUN curl -fSL -o /opt/tiktoken_cache/o200k_base.tiktoken \ - https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken && \ - curl -fSL -o /opt/tiktoken_cache/cl100k_base.tiktoken \ - https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken - -# Verify the harmony encoding loads from the pre-downloaded files -RUN python3 -c "\ -from openai_harmony import load_harmony_encoding, HarmonyEncodingName; \ -load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS); \ -print('openai_harmony encoding loaded OK')" diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 88abbfd..baa0e10 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -1,52 +1,48 @@ # NVFlow Container Images -NVFlow uses five container images, all designed to run fully offline on -air-gapped Slurm clusters. Four are **built locally** from the -self-contained Dockerfiles in this directory; the fifth (`sglang`) is pulled -as-is from Docker Hub. The Dockerfiles are build recipes β€” running -`docker build` against each one on a connected host produces the actual -images. +NVFlow uses six worker images plus an optional launcher. Five +(`nemo-rl`, `nemo-gym`, `nemo-skills`, `vllm`, `vllm-grpo`) are **built locally** +from the self-contained Dockerfiles in this directory, as is the optional +`nvflow-client` launcher; only `sglang` is **pulled as-is**. The Dockerfiles are +build recipes β€” running `docker build` against each one on a connected host +produces the actual images. For complete documentation β€” build instructions, sanity checks, deployment steps, air-gapped design rationale, and rebuild guidance β€” see **[docker_instructions.md](docker_instructions.md)**. -## Quick Start +> **Air-gap.** All six images run fully offline, including the `training` stage: +> `nemo-rl` bakes one NeMo-Gym venv per component at build time, so nothing is +> resolved or downloaded at job runtime and no Gym source mount is needed. +> `UV_OFFLINE` is left **unset** by policy β€” it keeps a dev-mode escape hatch, not +> because any stage needs the network. See +> [`docs/development/nemo-rl-gym.md`](../docs/development/nemo-rl-gym.md) for the +> trainer/Gym details. -```bash -# Requires `docker login nvcr.io` for the NGC registry (nemo-rl base image) -docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:v0.6.0 . -docker build -f dockerfiles/Dockerfile.nemo-skills -t nvflow-nemo-skills:0229040 . -docker build -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:v0.18.1 . -docker build -f dockerfiles/Dockerfile.vllm-grpo -t nvflow-vllm-grpo:v0.17.1 . +## Building -# Multi-arch builds (amd64 + arm64) β€” push directly to a registry -REGISTRY= -docker buildx build --platform linux/amd64,linux/arm64 \ - -f dockerfiles/Dockerfile.vllm -t $REGISTRY/nvflow-vllm:v0.18.1 --push . -docker buildx build --platform linux/amd64,linux/arm64 \ - -f dockerfiles/Dockerfile.vllm-grpo -t $REGISTRY/nvflow-vllm-grpo:v0.17.1 --push . - -# sglang β€” pull directly, no custom Dockerfile needed -docker pull lmsysorg/sglang:v0.5.10.post1 -``` +The full build commands β€” single-arch, multi-arch (`buildx` + QEMU), the +`sglang` pull, sanity checks, and `.sqsh` conversion β€” are in +**[docker_instructions.md](docker_instructions.md)** (the authoritative +build/deploy reference). The images and their version pins are below. ## Images | Image | Base | Purpose | |-------|------|---------| -| `nvflow-nemo-rl` | `nvcr.io/nvidia/nemo-rl:v0.6.0` | SFT, GRPO training, collect_rollouts, compute_rewards | +| `nvflow-nemo-rl` | `nvcr.io/nvidia/nemo-rl:v0.7.0` | SFT and GRPO `training`; NeMo-Gym venvs baked per component | | `nvflow-nemo-skills` | `ubuntu:22.04` | SDG pipeline, evaluation, data preparation | -| `nvflow-vllm` | `vllm/vllm-openai:v0.18.1` | Standalone vLLM inference (SDG, eval) β€” multi-arch (amd64 + arm64) | -| `nvflow-vllm-grpo` | `vllm/vllm-openai:v0.17.1` | Standalone vLLM inference (GRPO rollouts, judge) β€” multi-arch | +| `nvflow-vllm` | `vllm/vllm-openai:v0.22.0` | Standalone vLLM inference (SDG, eval) β€” multi-arch (amd64 + arm64) | +| `nvflow-vllm` (`v0.20.0*` tag) | `vllm/vllm-openai:v0.20.0` | Standalone vLLM inference (GRPO rollouts, judge) β€” multi-arch. Same `Dockerfile.vllm`, built with `--build-arg VLLM_VERSION=v0.20.0` | +| `nvflow-nemo-gym` | `python:3.12-slim` | CPU-only Gym-only GRPO stages (prepare_data, prefetch_cache, collect_rollouts, compute_rewards); finance Gym venvs baked | +| `nvflow-client` | pinned `ubuntu:24.04` | Optional launcher: drive `nflow` over an SSH tunnel (airgap/off-cluster); Python 3.12 + CLI + venv baked | | `sglang` | `lmsysorg/sglang:v0.5.10.post1` | SGLang inference server (pulled as-is) | ## Version Pins | Build Arg | Default | Where to find the right value | |-----------|---------|-------------------------------| -| `BASE_IMAGE` (nemo-rl) | `nvcr.io/nvidia/nemo-rl:v0.6.0` | [NGC NeMo-RL tags](https://catalog.ngc.nvidia.com) | -| `NEMO_SKILLS_COMMIT` | `022904023ad7a83a87662a313cf72e7df5891d55` (`0229040`) | Should match across `Dockerfile.nemo-skills` and `Dockerfile.nemo-rl` | -| `NEMO_GYM_BRANCH` | `ude/finance-sec-search-v2` | NeMo-Gym branch with finance agent | -| `VLLM_VERSION` (vllm) | `v0.18.1` | [vLLM releases](https://github.com/vllm-project/vllm/releases) | -| `VLLM_VERSION` (vllm-grpo) | `v0.17.1` | Pinned to match NeMo-RL v0.6.0 colocated vLLM | +| `NEMO_SKILLS_COMMIT` | `e06c9b90…` (image tag `v1.1.2`) | Should match `Dockerfile.nemo-skills` and `pyproject.toml` | +| `GYM_REF` (nemo-gym, nemo-rl) | `33ef60369…` | A commit on upstream NeMo-Gym `main`; keep both images on the same one | +| `VLLM_VERSION` (vllm) | `v0.22.0` | [vLLM releases](https://github.com/vllm-project/vllm/releases) | +| `VLLM_VERSION` (vllm-grpo) | `v0.20.0` | Pinned to match NeMo-RL v0.7.0 colocated vLLM | diff --git a/dockerfiles/docker_instructions.md b/dockerfiles/docker_instructions.md index fc2a989..762c6ad 100644 --- a/dockerfiles/docker_instructions.md +++ b/dockerfiles/docker_instructions.md @@ -1,31 +1,33 @@ # NVFlow Air-Gapped Docker Images Build, validate, and deploy the five NVFlow container images for use on -air-gapped Slurm clusters. Four of them are produced by running -`docker build` against the self-contained Dockerfiles in this directory; -the fifth (`sglang`) is pulled as-is from Docker Hub. All images are built -on a connected host (the only step that needs internet) and then run fully -offline on the cluster. +air-gapped Slurm clusters. Four of them (`nemo-rl`, `nemo-skills`, `vllm`, +`vllm-grpo`) are produced by running `docker build` against the self-contained +Dockerfiles in this directory; only `sglang` is pulled as-is. All custom images +are built on a connected host (the only step that needs internet) and then run +fully offline on the cluster. ## Images | Image | Base | Purpose | |---|---|---| -| `nvflow-nemo-rl` | `nvcr.io/nvidia/nemo-rl:v0.6.0` | SFT, GRPO training, collect_rollouts, compute_rewards | +| `nvflow-nemo-rl` | `nvcr.io/nvidia/nemo-rl:v0.7.0` | SFT and GRPO `training`; bakes one Gym venv per component so the trainer needs no network | | `nvflow-nemo-skills` | `ubuntu:22.04` | SDG pipeline, evaluation, data preparation, SEC data prep | -| `nvflow-vllm` | `vllm/vllm-openai:v0.18.1` | Standalone vLLM (SDG, eval) β€” multi-arch (amd64 + arm64) | -| `nvflow-vllm-grpo` | `vllm/vllm-openai:v0.17.1` | Standalone vLLM (GRPO rollouts, judge) β€” multi-arch | +| `nvflow-vllm` | `vllm/vllm-openai:v0.22.0` | Standalone vLLM (SDG, eval) β€” multi-arch (amd64 + arm64) | +| `nvflow-vllm` (`v0.20.0*` tag) | `vllm/vllm-openai:v0.20.0` | Standalone vLLM (GRPO rollouts, judge) β€” multi-arch. Same `Dockerfile.vllm` and repo as above; `--build-arg VLLM_VERSION=v0.20.0` | | `sglang` | `lmsysorg/sglang:v0.5.10.post1` | SGLang inference server (pulled as-is, no custom Dockerfile) | +Two more images build the same way but are only needed for specific paths: +`nvflow-nemo-gym` (CPU Gym worker, `Dockerfile.nemo-gym`, pinned by `GYM_REF`) for GRPO / DG-SDG, and `nvflow-client` (optional airgap-only launcher, `Dockerfile.nvflow`, multi-arch) for driving NVFlow over an `ssh_tunnel`. Both are covered in [containers.md](../docs/maintainers/containers.md). + ## Version pins | Build arg | Default | Where to find the right value | |---|---|---| -| `BASE_IMAGE` (nemo-rl) | `nvcr.io/nvidia/nemo-rl:v0.6.0` | [NGC NeMo-RL tags](https://catalog.ngc.nvidia.com) | -| `NEMO_SKILLS_COMMIT` | `022904023ad7a83a87662a313cf72e7df5891d55` (`0229040`) | Must match across `Dockerfile.nemo-skills` and `Dockerfile.nemo-rl` | -| `NEMO_GYM_BRANCH` | `ude/finance-sec-search-v2` | NeMo-Gym branch with finance agent | -| `VLLM_VERSION` (vllm) | `v0.18.1` | [vLLM releases](https://github.com/vllm-project/vllm/releases) | -| `VLLM_VERSION` (vllm-grpo) | `v0.17.1` | Pinned to match NeMo-RL v0.6.0 colocated vLLM | +| `NEMO_SKILLS_COMMIT` | `e06c9b90…` (image tag `v1.1.2`) | Must match `Dockerfile.nemo-skills` and `pyproject.toml` | +| `GYM_REF` (nemo-gym, nemo-rl) | `33ef60369…` | A commit on upstream NeMo-Gym `main`; keep both images on the same one | +| `VLLM_VERSION` (vllm) | `v0.22.0` | [vLLM releases](https://github.com/vllm-project/vllm/releases) | +| `VLLM_VERSION` (vllm-grpo) | `v0.20.0` | Pinned to match NeMo-RL v0.7.0 colocated vLLM | ## 1. Build @@ -45,10 +47,21 @@ below produce amd64 images. ```bash cd /path/to/nvflow -docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:v0.6.0 . -docker build -f dockerfiles/Dockerfile.nemo-skills -t nvflow-nemo-skills:0229040 . -docker build -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:v0.18.1 . -docker build -f dockerfiles/Dockerfile.vllm-grpo -t nvflow-vllm-grpo:v0.17.1 . +docker build --no-cache \ + -f dockerfiles/Dockerfile.nemo-skills -t nvflow-nemo-skills:v1.1.2 . + +# Airgapped trainer. Tag tracks the base version it extends. `docker build` pulls +# that base from nvcr.io, so `docker login nvcr.io` must have run first. +docker build -f dockerfiles/Dockerfile.nemo-rl -t nvflow-nemo-rl:v0.7.0 . + +# CPU-only Gym worker. Tag tracks the baked GYM_REF; bump it when GYM_REF moves. +docker build -f dockerfiles/Dockerfile.nemo-gym -t nvflow-nemo-gym:0.4.0 . + +# One Dockerfile builds both vLLM images; VLLM_VERSION picks the base tag, and +# both ship in the nvflow-vllm repo. +docker build -f dockerfiles/Dockerfile.vllm -t nvflow-vllm:v0.22.0 . +docker build -f dockerfiles/Dockerfile.vllm \ + --build-arg VLLM_VERSION=v0.20.0 -t nvflow-vllm:v0.20.0 . # sglang β€” pulled directly, no custom Dockerfile docker pull lmsysorg/sglang:v0.5.10.post1 @@ -70,7 +83,7 @@ Then build with an explicit `--platform`: # amd64 image from an arm64 host (most common cross-arch case for Slurm) docker buildx build --platform linux/amd64 \ -f dockerfiles/Dockerfile.vllm \ - -t nvflow-vllm:v0.18.1 \ + -t nvflow-vllm:v0.22.0 \ --load . ``` @@ -80,13 +93,15 @@ host when possible. ### linux/arm64 single-arch build -Only the two vLLM images are arm64-friendly today. From an arm64 host the -plain `docker build` works; from an amd64 host, use `buildx` with QEMU: +The custom images are all built multi-arch (see below); this single-platform +recipe is for testing one arch in isolation, and works for any of them by +swapping `-f`. From an arm64 host plain `docker build` works; from an amd64 +host, use `buildx` with QEMU: ```bash docker buildx build --platform linux/arm64 \ -f dockerfiles/Dockerfile.vllm \ - -t nvflow-vllm:v0.18.1-arm64 \ + -t nvflow-vllm:v0.22.0-arm64 \ --load . ``` @@ -94,25 +109,61 @@ docker buildx build --platform linux/arm64 \ ### Multi-arch build (amd64 + arm64) β€” push to registry -For `vllm` and `vllm-grpo`, build for both architectures and push the manifest -list in one shot. Multi-arch builds **must** push to a registry β€” the local -Docker image store can't hold a manifest list, so `--load` is not an option: +Build for both architectures and push the manifest list in one shot. Multi-arch +builds **must** push to a registry β€” the local Docker image store can't hold a +manifest list, so `--load` is not an option. + +Needs QEMU (above) and a `docker-container` builder β€” the default `docker` +driver cannot build multiple platforms: + +```bash +docker buildx create --name nvflow --driver docker-container --use +docker buildx inspect --bootstrap +``` ```bash REGISTRY= +# Airgapped trainer. Tag tracks the base version it extends. +docker buildx build --platform linux/amd64,linux/arm64 \ + -f dockerfiles/Dockerfile.nemo-rl \ + -t $REGISTRY/nvflow-nemo-rl:v0.7.0 \ + --provenance=false --sbom=false --push . + +# CPU-only Gym worker. Tag tracks the baked GYM_REF; bump it when GYM_REF moves. +docker buildx build --platform linux/amd64,linux/arm64 \ + -f dockerfiles/Dockerfile.nemo-gym \ + -t $REGISTRY/nvflow-nemo-gym:0.4.0 \ + --provenance=false --sbom=false --push . + +docker buildx build --platform linux/amd64,linux/arm64 \ + -f dockerfiles/Dockerfile.vllm \ + -t $REGISTRY/nvflow-vllm:v0.22.0 \ + --provenance=false --sbom=false --push . + +# Same Dockerfile and repo; VLLM_VERSION picks the base tag. docker buildx build --platform linux/amd64,linux/arm64 \ -f dockerfiles/Dockerfile.vllm \ - -t $REGISTRY/nvflow-vllm:v0.18.1 \ + --build-arg VLLM_VERSION=v0.20.0 \ + -t $REGISTRY/nvflow-vllm:v0.20.0 \ + --provenance=false --sbom=false --push . + +# --no-cache is required: ARG CACHEBUST gates the dependency-override layer, so +# a warm cache reuses stale resolutions and skips the security floors. +docker buildx build --platform linux/amd64,linux/arm64 --no-cache \ + -f dockerfiles/Dockerfile.nemo-skills \ + -t $REGISTRY/nvflow-nemo-skills:v1.1.2 \ --provenance=false --sbom=false --push . +# Launcher, not a worker. Build from a committed tree: .baked_commit records +# `git rev-parse HEAD`, so uncommitted changes ship under the wrong provenance. docker buildx build --platform linux/amd64,linux/arm64 \ - -f dockerfiles/Dockerfile.vllm-grpo \ - -t $REGISTRY/nvflow-vllm-grpo:v0.17.1 \ + -f dockerfiles/Dockerfile.nvflow \ + -t $REGISTRY/nvflow-client:v1.1.2 \ --provenance=false --sbom=false --push . # Verify both architectures are in the manifest list -docker buildx imagetools inspect $REGISTRY/nvflow-vllm:v0.18.1 +docker buildx imagetools inspect $REGISTRY/nvflow-vllm:v0.22.0 ``` `--provenance=false --sbom=false` keeps the manifest list compatible with @@ -133,50 +184,46 @@ image will not work in production. ### nemo-rl -```bash -IMAGE=nvflow-nemo-rl:v0.6.0 - -# A. uv works offline (paths relocated out of /root) -docker run --rm -e UV_OFFLINE=true $IMAGE bash -c \ - "uv python list --only-installed | grep 3.12" -# Expect: cpython-3.12.x at /opt/uv-python/... +> Required for the default release: `nvflow-nemo-rl` is built from +> `Dockerfile.nemo-rl`, and these checks are what prove its baked Gym venvs are +> usable offline. See +> [`docs/development/nemo-rl-gym.md`](../docs/development/nemo-rl-gym.md) for the +> image internals. -# B. main venv has no stale /root/.local references -docker run --rm $IMAGE bash -c ' - grep -rl "/root/.local" \ - /opt/nemo_rl_venv/pyvenv.cfg \ - /opt/ray_venvs/*/pyvenv.cfg \ - /opt/nemo-rl/3rdparty/Gym-workspace/Gym/.venv/pyvenv.cfg \ - 2>/dev/null || echo "All clean"' -# Expect: All clean - -# C. all 6 Gym component venvs are symlinked -docker run --rm $IMAGE bash -c ' - GYM=/opt/nemo-rl/3rdparty/Gym-workspace/Gym - for c in \ - resources_servers/equivalence_llm_judge \ - resources_servers/finance_sec_search \ - responses_api_agents/simple_agent \ - responses_api_agents/finance_agent \ - responses_api_models/openai_model \ - responses_api_models/vllm_model; do - [ -L "$GYM/$c/.venv" ] && echo "OK: $c" || echo "MISSING: $c" - done' -# Expect: 6x "OK: ..." +```bash +IMAGE=nvflow-nemo-rl:v0.7.0 + +# A. every Gym component venv is baked +docker run --rm $IMAGE bash -c \ + 'find /opt/gym_venvs -maxdepth 3 -name .venv | sort' +# Expect: 7 paths β€” resources_servers/{equivalence_llm_judge,finance_sec_search, +# format_verification}, responses_api_agents/{finance_agent,simple_agent}, +# responses_api_models/{openai_model,vllm_model} + +# B. Gym imports with no network and no uv resolve +docker run --rm --network=none -e UV_OFFLINE=true $IMAGE bash -c \ + '/opt/ray_venvs/nemo_rl.environments.nemo_gym.NemoGym/bin/python -c \ + "import nemo_gym; print(\"nemo_gym OK\")"' +# Expect: nemo_gym OK + +# C. /opt/NeMo-RL symlink (scripts/convert_checkpoint_to_hf.sh cd's to it) +docker run --rm $IMAGE bash -c \ + 'cd /opt/NeMo-RL && ls examples/converters/convert_dcp_to_hf.py' +# Expect: examples/converters/convert_dcp_to_hf.py # D. uvicorn pin (timeout_worker_healthcheck kwarg required by Gym servers) docker run --rm $IMAGE bash -c ' - /opt/nemo-rl/3rdparty/Gym-workspace/Gym/.venv/bin/python -c " + /opt/gym_venvs/resources_servers/finance_sec_search/.venv/bin/python -c " import uvicorn, inspect assert \"timeout_worker_healthcheck\" in inspect.signature(uvicorn.run).parameters, uvicorn.__version__ print(\"uvicorn\", uvicorn.__version__, \"OK\")"' -# Expect: uvicorn 0.37.x OK +# Expect: uvicorn 0.52.x OK ``` ### nemo-skills ```bash -IMAGE=nvflow-nemo-skills:0229040 +IMAGE=nvflow-nemo-skills:v1.1.2 # A. tiktoken pre-cache loads offline docker run --rm --network=none -e HF_HUB_OFFLINE=1 $IMAGE bash -c ' @@ -202,7 +249,7 @@ docker run --rm $IMAGE bash -c ' Run the same set against both images: ```bash -for IMAGE in nvflow-vllm:v0.18.1 nvflow-vllm-grpo:v0.17.1; do +for IMAGE in nvflow-vllm:v0.22.0 nvflow-vllm:v0.20.0; do echo "=== $IMAGE ===" # A. tiktoken encoding files present @@ -230,15 +277,17 @@ Push each image, then `enroot import` from the registry on the cluster: ```bash REGISTRY= -docker tag nvflow-nemo-rl:v0.6.0 $REGISTRY/nvflow-nemo-rl:v0.6.0 -docker tag nvflow-nemo-skills:0229040 $REGISTRY/nvflow-nemo-skills:0229040 -docker tag nvflow-vllm:v0.18.1 $REGISTRY/nvflow-vllm:v0.18.1 -docker tag nvflow-vllm-grpo:v0.17.1 $REGISTRY/nvflow-vllm-grpo:v0.17.1 - -docker push $REGISTRY/nvflow-nemo-rl:v0.6.0 -docker push $REGISTRY/nvflow-nemo-skills:0229040 -docker push $REGISTRY/nvflow-vllm:v0.18.1 -docker push $REGISTRY/nvflow-vllm-grpo:v0.17.1 +docker tag nvflow-nemo-rl:v0.7.0 $REGISTRY/nvflow-nemo-rl:v0.7.0 +docker tag nvflow-nemo-gym:0.4.0 $REGISTRY/nvflow-nemo-gym:0.4.0 +docker tag nvflow-nemo-skills:v1.1.2 $REGISTRY/nvflow-nemo-skills:v1.1.2 +docker tag nvflow-vllm:v0.22.0 $REGISTRY/nvflow-vllm:v0.22.0 +docker tag nvflow-vllm:v0.20.0 $REGISTRY/nvflow-vllm:v0.20.0 + +docker push $REGISTRY/nvflow-nemo-rl:v0.7.0 +docker push $REGISTRY/nvflow-nemo-gym:0.4.0 +docker push $REGISTRY/nvflow-nemo-skills:v1.1.2 +docker push $REGISTRY/nvflow-vllm:v0.22.0 +docker push $REGISTRY/nvflow-vllm:v0.20.0 ``` Then on the cluster (typically a CPU partition): @@ -247,11 +296,14 @@ Then on the cluster (typically a CPU partition): CONTAINER_DIR= REGISTRY= +# Name the output -.sqsh to match what scripts/setup_containers.sh +# produces, so either staging method drops in to the same my_cluster.yaml. enroot import \ - --output $CONTAINER_DIR/nvflow-nemo-rl-v0.6.0.sqsh \ - "docker://$REGISTRY/nvflow-nemo-rl:v0.6.0" + --output $CONTAINER_DIR/nemo-skills-v1.1.2.sqsh \ + "docker://$REGISTRY/nvflow-nemo-skills:v1.1.2" -# Repeat for nemo-skills, vllm, vllm-grpo, and (optionally) sglang. +# Repeat for vllm, vllm-grpo, nemo-gym, nemo-rl. sglang imports directly: +# docker://lmsysorg/sglang:v0.5.10.post1 ``` If the cluster authenticates to your registry, drop credentials into @@ -269,14 +321,14 @@ load the tarball into the local Docker daemon, then import via `dockerd://`: ```bash # On the build host -docker save nvflow-nemo-rl:v0.6.0 | gzip > nvflow-nemo-rl-v0.6.0.tar.gz +docker save nvflow-nemo-skills:v1.1.2 | gzip > nvflow-nemo-skills-v1.1.2.tar.gz # Transfer the .tar.gz to the cluster (scp / rsync / sneakernet) # On the cluster (requires a Docker daemon accessible to your user) -gunzip -c nvflow-nemo-rl-v0.6.0.tar.gz | docker load +gunzip -c nvflow-nemo-skills-v1.1.2.tar.gz | docker load enroot import \ - --output $CONTAINER_DIR/nvflow-nemo-rl-v0.6.0.sqsh \ - dockerd://nvflow-nemo-rl:v0.6.0 + --output $CONTAINER_DIR/nemo-skills-v1.1.2.sqsh \ + dockerd://nvflow-nemo-skills:v1.1.2 ``` > `enroot import` natively supports only `docker://` (remote registry), @@ -291,14 +343,14 @@ enroot import \ path, which breaks for registries where the host itself contains a path (e.g. `nvcr.io/`). Use `#` to separate host from image path: ```bash - enroot import --output nvflow-vllm-v0.18.1.sqsh \ - "docker://nvcr.io#/nvflow-vllm:v0.18.1" + enroot import --output vllm-v0.22.0.sqsh \ + "docker://nvcr.io#/nvflow-vllm:v0.22.0" ``` - **Filename colon.** `enroot` writes the Docker tag separator (`:`) literally into the output filename. Either pass `--output` with a shell-safe name (as above) or rename after import: ```bash - mv "nvflow-nemo-rl:v0.6.0.sqsh" nvflow-nemo-rl-v0.6.0.sqsh + mv "nvflow-nemo-skills:v1.1.2.sqsh" nemo-skills-v1.1.2.sqsh ``` ## 4. Cluster config (`my_cluster.yaml`) @@ -312,10 +364,12 @@ operation. ```yaml containers: - nemo-rl: /nvflow-nemo-rl-v0.6.0.sqsh - nemo-skills: /nvflow-nemo-skills-0229040.sqsh - vllm: /nvflow-vllm-v0.18.1.sqsh - vllm-grpo: /nvflow-vllm-grpo-v0.17.1.sqsh + # Filenames are -.sqsh, as produced by setup_containers.sh. + nemo-rl: /nemo-rl-v0.7.0.sqsh + nemo-skills: /nemo-skills-v1.1.2.sqsh + vllm: /vllm-v0.22.0.sqsh + vllm-grpo: /vllm-grpo-v0.20.0.sqsh + nemo-gym: /nemo-gym-0.4.0.sqsh # sglang: /sglang-v0.5.10.post1.sqsh ``` @@ -330,8 +384,10 @@ env_vars: - HF_DATASETS_OFFLINE=1 - TRANSFORMERS_OFFLINE=1 - # Disable uv package and Python interpreter downloads. - - UV_OFFLINE=true + # Disable uv package and Python interpreter downloads. Left UNSET: every image + # bakes the venvs it needs, so no stage resolves at runtime either way, and + # unset keeps a dev-mode escape hatch. + # - UV_OFFLINE=true # Point tiktoken / openai_harmony at the cache baked into the images. # Required for nemo-skills and nemo-rl (vllm/vllm-grpo set them as ENV). @@ -340,35 +396,29 @@ env_vars: - TIKTOKEN_ENCODINGS_BASE=/opt/tiktoken_cache ``` -### Don't bind-mount NeMo-RL or NeMo-Gym source over the image paths - -The air-gapped `nvflow-nemo-rl` image already contains NeMo-Gym venv -at `/opt/NeMo-RL/3rdparty/Gym-workspace/Gym/.venv` (sanity check **C** in -section 2 verifies this). GRPO stages source that venv via -`installation_command: source .../Gym/.venv/bin/activate` before running. - -Older dev-mode `my_cluster.yaml` templates often include host source overlays -like: +### NeMo-RL / NeMo-Gym: trainer image and Gym source -```yaml -mounts: - # DO NOT use these with the air-gapped image β€” they shadow the baked .venv - # - /RL:/opt/NeMo-RL - # - /Gym:/opt/NeMo-RL/3rdparty/Gym-workspace/Gym -``` +The `training` stage runs on `nvflow-nemo-rl`, built here from +`Dockerfile.nemo-rl`. It extends the NeMo-RL base with the Gym source at +`GYM_REF` and one prebuilt venv per Gym component under `/opt/gym_venvs`, so +nothing resolves at runtime and **no Gym source mount is required**. -These bind-mounts hide the baked `.venv` symlink and the `installation_command` -fails with `No such file or directory` β€” breaking `prepare_data`, -`collect_rollouts`, `compute_rewards`, and `training` for GRPO. Only add -these mounts if you are deliberately iterating on NeMo-RL/Gym source against a -host `.venv` you've built to be ABI-compatible with the image. +Do not bind-mount a host Gym or NeMo-RL clone over +`/opt/nemo-rl/3rdparty/Gym-workspace/Gym` in production β€” it shadows the baked +source and venvs and breaks the GRPO stages. That mount is a dev-mode-only tool, +and it is the one case where `uv` resolves at runtime, so it needs `UV_OFFLINE` +unset plus a reachable pypi mirror. -### Don't enable this in offline mode +The Gym-only GRPO stages (`prepare_data`, `prefetch_cache`, `collect_rollouts`, +`compute_rewards`) run on the separate `nvflow-nemo-gym` image, also with baked +per-component venvs and no mount. -```yaml -# - NRL_FORCE_REBUILD_VENVS=true # forces Ray workers to re-resolve via uv - # (requires internet; will fail under air-gap) -``` +For a fully-airgapped trainer with no runtime `uv` resolve, build the custom +image from [`Dockerfile.nemo-rl`](Dockerfile.nemo-rl) (bakes the Gym venvs) and +drop the Gym mount. That image needs no network at job time on its own, so +`UV_OFFLINE` still stays **unset** by policy: leaving it unset is what lets a +developer mount local Gym source and have `uv` resolve it. See +[`docs/development/nemo-rl-gym.md`](../docs/development/nemo-rl-gym.md). ## Notes for one-time / connected-node operations @@ -384,8 +434,8 @@ air-gapped cluster: | `workflow-5 step-4 prepare_data` (GRPO) | Only if `should_download: true`; default `should_download: false` requires no internet. | For these stages, temporarily clear the three HF flags -(`HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE`, `TRANSFORMERS_OFFLINE`). Keep -`UV_OFFLINE=true` set β€” `uv` should never need to resolve packages at runtime. +(`HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE`, `TRANSFORMERS_OFFLINE`). `UV_OFFLINE` +stays unset as always; none of these stages invoke `uv`. Note: `huggingface_hub` interprets `TRANSFORMERS_OFFLINE=1` as `HF_HUB_OFFLINE=1`, so all three need to be off (or unset) for HF dataset diff --git a/docs/architecture/ARCHITECTURE.md b/docs/ARCHITECTURE.md similarity index 88% rename from docs/architecture/ARCHITECTURE.md rename to docs/ARCHITECTURE.md index 1d58a20..dfc3fa3 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,7 +1,5 @@ # NVFlow Architecture -> **Version:** 1.0 -> **Last Updated:** January 21, 2026 > **Purpose:** Comprehensive architectural overview of the NVFlow orchestration framework --- @@ -361,11 +359,11 @@ sequenceDiagram NemoSkills->>Slurm: Submit job with dependencies Slurm-->>NemoSkills: Job ID NemoSkills-->>Stage: Job submitted - Stage-->>WorkflowRunner: Stage complete + Stage-->>WorkflowRunner: Stage submitted end - WorkflowRunner-->>CLI: All stages complete - CLI-->>User: βœ… Workflow Complete! + WorkflowRunner-->>CLI: All stages submitted + CLI-->>User: βœ… Workflow Submitted ``` ### Dependency Resolution @@ -437,8 +435,9 @@ graph TB TBS --> DataPrep DGS -.-> DataPrep DataPrep --> SFT + DataPrep --> GRPO SFT --> Eval - SFT --> GRPO + SFT -.-> GRPO GRPO --> Eval style SEC fill:#e3f2fd @@ -450,6 +449,8 @@ graph TB style GRPO fill:#fff3e0 ``` +Dashed edges are optional. GRPO starts from the base HuggingFace checkpoint in the shipped configs (`hf_model_path: /hf_models/Qwen/Qwen3-4B`), so SFT is not a prerequisite for it β€” point `hf_model_path` at an SFT checkpoint only if you want to chain the two. + ### Workflow Breakdown #### **Workflow 1: Download SEC Filings** @@ -479,14 +480,14 @@ Models: GPT-OSS-120B, Qwen3-14B ``` Stages: 1. dg_sdg_preprocess - Preprocess filings - 2. document_grounded_qa_generation - Generate verified Q&A - 3. genselect_answers - Self-consistency check - 4. evaluate_answers - Quality evaluation - 5. aggregate_answers - Combine results - 6. difficulty_estimation - Stratify by difficulty - 7. document_grounded_data - Prepare training data - -Output: ~800K Q&A pairs (stratified) + 2. generate_verified_questions - Question generation + verification + 3. generate_answers - Answer generation (multi-rollout) + 4. gym_genselect_answers - Self-consistency selection + 5. evaluate_answers - Quality evaluation + 6. aggregate_answers - Combine multi-seed results + 7. dgsdg_post_process - Clean + rename β†’ final_result.jsonl + +Output: ~800K Q&A pairs in final_result.jsonl GPU: 8 GPUs Models: Qwen3 family (14B-235B) ``` @@ -497,13 +498,15 @@ Stages: 1. data_transformation - Convert to training format 2. prepare_for_sft - Format for NeMo 3. train_validation_split - Split dataset - 4. sequence_length_grouping - [Optional] Group by length + 4. sequence_length_grouping - Group by length 5. training - Multi-node training - 6. convert_to_messages - [Optional] Post-processing + 6. eval - Evaluate checkpoints on benchmarks + + Qwen3 configs insert convert_to_messages between training and eval. GPU: 256 GPUs (32 nodes Γ— 8 GPUs) Model: Qwen3-14B -Parallelism: TP=2, PP=1, CP=2 +Parallelism: TP=4, PP=1, CP=8 ``` #### **Workflow 5: Evaluation** @@ -521,7 +524,7 @@ Benchmarks: Financial reasoning tasks #### **Workflow 6: GRPO RL Training** ``` Stages: - 1. validate_questions - Validate format + deduplicate + 1. validate_questions - Regex prefilter + LLM validity classifier 2. data_transformation - SDG cleanup to model-agnostic schema 3. apply_prompt_template - Apply prompt template + extract answer 4. convert_to_responses_api - Convert to NeMo-Gym Responses API format @@ -534,7 +537,7 @@ Stages: Output: RL-trained model + eval results GPU: 16 GPUs (2 nodes for demo), 64 GPUs (8 nodes for production) -Model: Qwen3-4B dense (demo, FSDP v2), Qwen3-30B-A3B MoE (production, Megatron) +Model: Qwen3-4B dense (demo β€” equivalence_llm_judge: FSDP v2 @ 32K; finance_sec_search: Megatron TP2Γ—CP8 @ 131K), Qwen3-30B-A3B MoE (production, Megatron) ``` ### Finance Recipe Component Diagram @@ -544,12 +547,14 @@ graph TB subgraph "Finance Recipe Structure" direction TB - subgraph "Stages (42 total)" + subgraph "Stage modules (23 total)" direction LR - SDG[SDG Stages
12 stages] - SFT[SFT Stages
4 stages] - Eval[Eval Stages
2 stages] - RL[RL Stages
10 stages] + SDG[SDG
6 modules] + RL[RL
8 modules] + SFT[SFT
4 modules] + Eval[Eval
2 modules] + DL[Download
1 module] + Shared[Shared
2 modules] end subgraph "Workflows (6 total)" @@ -557,7 +562,7 @@ graph TB W2[template-sdg
6 stages] W3[document-sdg
7 stages] W4[sft
6 stages] - W5[eval
9 stages] + W5[eval
7 stages] W6[grpo
10 stages] end @@ -567,12 +572,14 @@ graph TB P3[Evaluation Prompts] end - W1 -.-> SDG + W1 -.-> DL W2 -.-> SDG W3 -.-> SDG W4 -.-> SFT + W4 -.-> Shared W5 -.-> Eval W6 -.-> RL + W6 -.-> Shared SDG -.-> P1 SDG -.-> P2 @@ -583,6 +590,8 @@ graph TB style SFT fill:#bbdefb style Eval fill:#f8bbd0 style RL fill:#fff3e0 + style DL fill:#ede7f6 + style Shared fill:#eceff1 style W1 fill:#e1f5ff style W2 fill:#e1f5ff style W3 fill:#e1f5ff @@ -669,35 +678,41 @@ graph TB ``` β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Air-gapped container images (.sqsh format) β”‚ -β”‚ (built locally from dockerfiles/Dockerfile.* β€” see INSTALL.md)β”‚ +β”‚ Container images (.sqsh format) β”‚ +β”‚ (nemo-rl/nemo-skills/vllm/vllm-grpo/nemo-gym built from β”‚ +β”‚ dockerfiles/*; only sglang pulled as-is β€” see INSTALL.md) β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ nvflow-nemo-skills β”‚ β”‚ nvflow-vllm β”‚ β”‚ -β”‚ β”‚ (0229040) β”‚ β”‚ (v0.18.1) β”‚ β”‚ +β”‚ β”‚ (v1.1.2) β”‚ β”‚ (v0.22.0) β”‚ β”‚ β”‚ β”‚ SDG/eval/data prep β”‚ β”‚ SDG/eval inference β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ sglang (pulled) β”‚ β”‚ nvflow-nemo-rl β”‚ β”‚ -β”‚ β”‚ (v0.5.10.post1) β”‚ β”‚ (v0.6.0) β”‚ β”‚ -β”‚ β”‚ SDG inference β”‚ β”‚ SFT/GRPO + Gym venv β”‚ β”‚ +β”‚ β”‚ (v0.5.10.post1) β”‚ β”‚ (v0.7.0) β”‚ β”‚ +β”‚ β”‚ SDG inference β”‚ β”‚ SFT/GRPO trainer β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ nvflow-vllm-grpo β”‚ β”‚ -β”‚ β”‚ (v0.17.1) β”‚ β”‚ -β”‚ β”‚ GRPO rollout/judge β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ nvflow-vllm β”‚ β”‚ nvflow-nemo-gym β”‚ β”‚ +β”‚ β”‚ (v0.20.0 tag) β”‚ β”‚ (0.4.0) β”‚ β”‚ +β”‚ β”‚ GRPO rollout/judge β”‚ β”‚ CPU Gym-only stages β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +The five `nvflow-*` images each require a `docker build` on a connected host before use, then run fully offline; only `sglang` can be pulled directly. See [`dockerfiles/docker_instructions.md`](../dockerfiles/docker_instructions.md). + +``` ↓ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Shared Filesystem Mounts β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ /workspace β†’ /lustre/.../workspace β”‚ +β”‚ /workspace β†’ writable data (outputs, HF cache) β”‚ β”‚ /hf_models β†’ /lustre/.../models/hf_models β”‚ -β”‚ /outputs β†’ /lustre/.../outputs β”‚ +β”‚ /nemo_run/code β†’ recipe code (nemo-run packaged) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` @@ -876,7 +891,7 @@ graph TB ### Storage Layout ``` -/lustre/fsw/.../workspace/nvflow/ +/workspace/nvflow/ β”‚ β”œβ”€β”€ cluster_configs/ # Cluster configuration β”‚ β”œβ”€β”€ containers.yaml # Container definitions @@ -921,26 +936,8 @@ graph TB ## Summary -### Key Architectural Highlights - -1. **Modular Design**: Clear separation between framework, recipes, and infrastructure -2. **Hierarchical Organization**: Recipe β†’ Workflow β†’ Stage provides natural organization -3. **Declarative Configuration**: YAML-based configs with inheritance support -4. **Flexible Execution**: CLI, Python API, and programmatic interfaces -5. **Cluster Native**: First-class Slurm integration with dependency management -6. **Extensible**: Easy to add new recipes, workflows, and stages -7. **Built on NeMo**: Leverages NVIDIA's NeMo ecosystem (Skills, RL, Framework) - -### Design Benefits - -- **Reproducibility**: Version-controlled configs and deterministic execution -- **Reusability**: Stages can be shared across workflows and recipes -- **Scalability**: Seamless scaling from local development to multi-node clusters -- **Maintainability**: Clear structure and separation of concerns -- **Discoverability**: Registry pattern enables stage discovery and documentation - ---- +Three ideas carry most of the design: -**Document Version:** 1.0 -**Generated:** January 21, 2026 -**Repository:** nvflow +- **Recipe β†’ Workflow β†’ Stage.** Stages are the unit of reuse and are discovered through the registry, so they can be shared across workflows and recipes, and listed without being hardcoded anywhere. +- **Declarative, inheritable YAML.** A model config inherits a workflow base and patches it, which keeps runs version-controlled and reproducible. +- **Slurm-native submission.** Stages submit jobs with dependencies rather than executing inline, which is what lets the same config scale from a demo to a multi-node production run. diff --git a/docs/architecture/ARCHITECTURE_INDEX.md b/docs/architecture/ARCHITECTURE_INDEX.md deleted file mode 100644 index dc4dbd1..0000000 --- a/docs/architecture/ARCHITECTURE_INDEX.md +++ /dev/null @@ -1,435 +0,0 @@ -# NVFlow - Architecture Documentation Index - -> **Navigation guide for all architecture documentation** -> **Start here to find the right documentation for your needs** - ---- - -## πŸ“š Documentation Overview - -The NVFlow architecture is documented across multiple files, each serving a specific purpose. This index helps you find the right documentation quickly. - ---- - -## 🎯 Quick Navigation - -### I want to... - -| Goal | Document | Time | -|------|----------|------| -| **Get a quick overview** | [ARCHITECTURE_QUICK_REFERENCE.md](#quick-reference) | 5 min | -| **Understand the system deeply** | [ARCHITECTURE.md](#comprehensive-architecture) | 30 min | -| **View visual diagrams** | [diagrams/](#visual-diagrams) | 10 min | -| **Learn about diagrams** | [DIAGRAMS_SUMMARY.md](#diagrams-summary) | 10 min | -| **Get started with NVFlow** | [README.md](#main-readme) | 15 min | -| **Set up the cluster** | [INSTALL.md](#installation-guide) | 30 min | -| **Learn the finance recipe** | [docs/recipes/finance/](#finance-recipe-docs) | 45 min | - ---- - -## πŸ“– Document Descriptions - -### Quick Reference -**File:** [ARCHITECTURE_QUICK_REFERENCE.md](./ARCHITECTURE_QUICK_REFERENCE.md) -**Size:** ~5 KB -**Reading Time:** 5 minutes -**Best For:** Quick onboarding, cheat sheet, reference card - -**Contents:** -- One-page architecture overview -- Core components table -- Common commands -- Quick stage creation guide -- Key features checklist -- Documentation map - -**When to Use:** -- First time learning about NVFlow -- Need a quick reminder of concepts -- Looking for specific commands -- Want a printable reference - ---- - -### Comprehensive Architecture -**File:** [ARCHITECTURE.md](./ARCHITECTURE.md) -**Size:** ~26 KB -**Reading Time:** 30 minutes -**Best For:** Deep understanding, system design, contribution - -**Contents:** -1. High-level architecture with diagrams -2. System overview and design principles -3. Core framework components (detailed) -4. Hierarchical organization (Recipe β†’ Workflow β†’ Stage) -5. Complete execution flow with sequence diagrams -6. Finance recipe architecture (all 42 stages) -7. Deployment architecture and topology -8. Technology stack and integrations -9. Data flow diagrams - -**When to Use:** -- Need comprehensive system understanding -- Planning to contribute to the codebase -- Designing new recipes or workflows -- Troubleshooting complex issues -- Presenting architecture to stakeholders - ---- - -### Diagrams Summary -**File:** [DIAGRAMS_SUMMARY.md](./DIAGRAMS_SUMMARY.md) -**Size:** ~12 KB -**Reading Time:** 10 minutes -**Best For:** Understanding available diagrams, diagram usage guide - -**Contents:** -- Overview of all 5 diagrams -- Diagram details and use cases -- Audience-specific recommendations -- Question-to-diagram mapping -- Diagram statistics -- Rendering examples -- Update guidelines - -**When to Use:** -- Want to know what diagrams are available -- Need to choose the right diagram -- Want to render diagrams in different formats -- Planning to create new diagrams - ---- - -### Visual Diagrams -**Location:** [diagrams/](../diagrams/) -**Format:** Mermaid (.mmd files) -**Count:** 5 diagrams + README -**Best For:** Visual learners, presentations, documentation - -**Available Diagrams:** - -1. **[architecture-overview.mmd](../diagrams/architecture-overview.mmd)** - - High-level system architecture - - All major components and relationships - - 5 layers: UI, Core, Recipes, Infrastructure, Storage - -2. **[finance-pipeline.mmd](../diagrams/finance-pipeline.mmd)** - - Complete finance recipe pipeline - - All 6 workflows with 42 stages - - Data flow from SEC filings to evaluation - -3. **[execution-flow.mmd](../diagrams/execution-flow.mmd)** - - Runtime execution sequence diagram - - User command to job completion - - 4 phases: Init, Validate, Execute, Monitor - -4. **[component-architecture.mmd](../diagrams/component-architecture.mmd)** - - Class diagram of core framework - - BaseStage, StageRegistry, WorkflowRunner - - Relationships and dependencies - -5. **[deployment-architecture.mmd](../diagrams/deployment-architecture.mmd)** - - Infrastructure and deployment topology - - Local machine to Slurm cluster - - Compute nodes, storage, containers - -**Viewing Options:** -- Online: https://mermaid.live/ -- VS Code: Mermaid Preview extension -- CLI: `mmdc -i diagram.mmd -o diagram.png` -- GitHub: Native rendering - -**When to Use:** -- Need visual understanding -- Creating presentations -- Onboarding new team members -- Documentation in other systems - ---- - -### Main README -**File:** [README.md](../../README.md) -**Size:** ~10 KB -**Reading Time:** 15 minutes -**Best For:** Getting started, understanding concepts, running workflows - -**Contents:** -- Project overview and key features -- Core concepts (Recipe, Workflow, Stage) -- Folder structure explanation -- Installation instructions -- Quick start examples -- CLI commands reference -- Development guide - -**When to Use:** -- First time using NVFlow -- Need to understand basic concepts -- Want to run your first workflow -- Looking for CLI command syntax - ---- - -### Installation Guide -**File:** [INSTALL.md](../../INSTALL.md) -**Size:** ~8 KB -**Reading Time:** 30 minutes (including setup) -**Best For:** Cluster setup, container configuration, troubleshooting - -**Contents:** -1. Prerequisites (uv, yq, enroot) -2. Container setup (automated script) -3. Model download instructions -4. Cluster configuration -5. Verification steps -6. Troubleshooting guide - -**When to Use:** -- Setting up NVFlow for the first time -- Configuring a new cluster -- Troubleshooting installation issues -- Understanding container requirements - ---- - -### Finance Recipe Docs -**Location:** [docs/recipes/finance/](../recipes/finance/) -**Size:** Multiple files (~20 KB total) -**Reading Time:** 45 minutes -**Best For:** Understanding finance recipe, running production pipelines - -**Main Files:** - -1. **[README.md](../recipes/finance/README.md)** - Recipe overview - - 6 workflows, 42 stages - - Pipeline architecture - - Getting started guide - - Command reference - -2. **[quick-start.md](../recipes/finance/quick-start.md)** - 30-min demo - - Hands-on tutorial with 7 companies - - Step-by-step instructions - - Expected outputs - -3. **Workflow Guides** (in `workflows/`) - - 01-download-sec.md - - 02-template-based-sdg.md - - 03-document-grounded-sdg.md - - 04-sft.md - - 05-eval.md - - 06-grpo.md - - 06-finance-agent-eval.md - -4. **Stage Reference** (in `stages/`) - - Technical specifications for all 42 stages - - Input/output formats - - Configuration options - -5. **[troubleshooting.md](../recipes/finance/troubleshooting.md)** - - Common issues and solutions - - Debugging tips - -**When to Use:** -- Running the finance recipe -- Understanding SDG approaches -- Training financial reasoning models -- Troubleshooting finance-specific issues - ---- - -## πŸ—ΊοΈ Documentation Map (Visual) - -``` -NVFlow Documentation -β”‚ -β”œβ”€ πŸ“˜ Getting Started -β”‚ β”œβ”€ README.md ...................... Project overview & quick start -β”‚ β”œβ”€ INSTALL.md ..................... Cluster setup guide -β”‚ └─ ARCHITECTURE_QUICK_REFERENCE.md One-page cheat sheet -β”‚ -β”œβ”€ πŸ—οΈ Architecture -β”‚ β”œβ”€ ARCHITECTURE.md ................ Comprehensive architecture (26 KB) -β”‚ β”œβ”€ DIAGRAMS_SUMMARY.md ............ Diagram usage guide -β”‚ β”œβ”€ ARCHITECTURE_INDEX.md .......... This file -β”‚ └─ diagrams/ ...................... Visual diagrams (5 files) -β”‚ β”œβ”€ architecture-overview.mmd -β”‚ β”œβ”€ finance-pipeline.mmd -β”‚ β”œβ”€ execution-flow.mmd -β”‚ β”œβ”€ component-architecture.mmd -β”‚ β”œβ”€ deployment-architecture.mmd -β”‚ └─ README.md -β”‚ -β”œβ”€ 🍴 Recipes -β”‚ β”œβ”€ docs/recipes/finance/ .......... Finance recipe (production) - β”‚ β”‚ β”œβ”€ README.md ................... Recipe overview - β”‚ β”‚ β”œβ”€ quick-start.md .............. 30-min demo - β”‚ β”‚ β”œβ”€ workflows/ .................. 7 workflow guides - β”‚ β”‚ β”œβ”€ stages/ ..................... 42 stage specifications -β”‚ β”‚ └─ troubleshooting.md .......... Common issues -β”‚ β”‚ -β”‚ └─ nvflow/recipes/example/ ..... Example recipe (learning) -β”‚ -β”œβ”€ πŸ’» Code & development docs -β”‚ β”œβ”€ docs/development/console-ui.md .. Console UI guide (stage terminal output) -β”‚ └─ tests/README.md ................ Testing guide -β”‚ -└─ πŸ”§ Configuration - β”œβ”€ cluster_configs/ ............... Cluster configuration files - └─ pyproject.toml ................. Project dependencies -``` - ---- - -## πŸ‘₯ Audience-Specific Paths - -### For New Users -1. Start: [README.md](../../README.md) - Understand what NVFlow is -2. Quick ref: [ARCHITECTURE_QUICK_REFERENCE.md](./ARCHITECTURE_QUICK_REFERENCE.md) - Key concepts -3. Visual: [diagrams/architecture-overview.mmd](../diagrams/architecture-overview.mmd) - See the big picture -4. Try it: [docs/recipes/finance/quick-start.md](../recipes/finance/quick-start.md) - Run first workflow - -### For Data Scientists -1. Overview: [README.md](../../README.md) - Core concepts -2. Pipeline: [diagrams/finance-pipeline.mmd](../diagrams/finance-pipeline.mmd) - See data flow -3. Recipe: [docs/recipes/finance/README.md](../recipes/finance/README.md) - Finance pipeline -4. Run: [docs/recipes/finance/quick-start.md](../recipes/finance/quick-start.md) - Hands-on demo - -### For ML Engineers -1. Setup: [INSTALL.md](../../INSTALL.md) - Cluster configuration -2. Architecture: [ARCHITECTURE.md](./ARCHITECTURE.md) - System design -3. Execution: [diagrams/execution-flow.mmd](../diagrams/execution-flow.mmd) - Runtime behavior -4. Troubleshoot: [docs/recipes/finance/troubleshooting.md](../recipes/finance/troubleshooting.md) - -### For Software Engineers / Stage Authors -1. Components: [diagrams/component-architecture.mmd](../diagrams/component-architecture.mmd) - Class structure -2. Deep dive: [ARCHITECTURE.md](./ARCHITECTURE.md) - Design patterns -3. Code: Browse `nvflow/core/` - Framework implementation -4. Extend: [README.md](../../README.md#-creating-a-stage) - Create new stages -5. Console UI: [docs/development/console-ui.md](../development/console-ui.md) - Terminal output in stage `execute()` methods - -### For DevOps/Infrastructure -1. Setup: [INSTALL.md](../../INSTALL.md) - Installation guide -2. Deployment: [diagrams/deployment-architecture.mmd](../diagrams/deployment-architecture.mmd) - Topology -3. Cluster: [ARCHITECTURE.md](./ARCHITECTURE.md#7-deployment-architecture) - Infrastructure details -4. Config: `cluster_configs/` - Configuration files - -### For System Architects -1. Overview: [ARCHITECTURE_QUICK_REFERENCE.md](./ARCHITECTURE_QUICK_REFERENCE.md) - Quick scan -2. Complete: [ARCHITECTURE.md](./ARCHITECTURE.md) - Full architecture -3. All diagrams: [diagrams/](../diagrams/) - Visual representations -4. Design: [ARCHITECTURE.md](./ARCHITECTURE.md#2-system-overview) - Design principles - ---- - -## πŸ” Finding Specific Information - -### Concepts & Terminology -- **Recipe, Workflow, Stage:** [README.md](../../README.md#-core-concepts) -- **Hierarchical organization:** [ARCHITECTURE.md](./ARCHITECTURE.md#4-hierarchical-organization) -- **Design patterns:** [ARCHITECTURE.md](./ARCHITECTURE.md#key-design-patterns) - -### How-To Guides -- **Create a stage:** [README.md](../../README.md#-creating-a-stage) -- **Console output in stages:** [docs/development/console-ui.md](../development/console-ui.md) - Use `console.status()`, `console.detail()`, etc. -- **Run a workflow:** [README.md](../../README.md#-quick-start) -- **Set up cluster:** [INSTALL.md](../../INSTALL.md) -- **Run finance recipe:** [docs/recipes/finance/quick-start.md](../recipes/finance/quick-start.md) - -### Technical Reference -- **CLI commands:** [README.md](../../README.md#-cli-commands) -- **Core components:** [ARCHITECTURE.md](./ARCHITECTURE.md#3-core-framework-components) -- **Finance stages:** [docs/recipes/finance/stages/](../recipes/finance/stages/) -- **API reference:** Code docstrings in `nvflow/core/` - -### Visual Diagrams -- **System overview:** [diagrams/architecture-overview.mmd](../diagrams/architecture-overview.mmd) -- **Data pipeline:** [diagrams/finance-pipeline.mmd](../diagrams/finance-pipeline.mmd) -- **Execution flow:** [diagrams/execution-flow.mmd](../diagrams/execution-flow.mmd) -- **Class structure:** [diagrams/component-architecture.mmd](../diagrams/component-architecture.mmd) -- **Infrastructure:** [diagrams/deployment-architecture.mmd](../diagrams/deployment-architecture.mmd) - ---- - -## πŸ“Š Documentation Statistics - -| Metric | Count | -|--------|-------| -| Total documentation files | 20+ | -| Architecture documents | 4 | -| Visual diagrams | 5 | -| Recipe guides | 10+ | -| Total pages (estimated) | 100+ | -| Total size | ~100 KB | - ---- - -## πŸ”„ Documentation Maintenance - -### When to Update - -| Change Type | Documents to Update | -|-------------|-------------------| -| New recipe | Architecture overview, diagrams | -| New stage | Recipe docs, pipeline diagram | -| Core framework change | ARCHITECTURE.md, component diagram | -| Infrastructure change | INSTALL.md, deployment diagram | -| New workflow | Recipe README, workflow guide | - -### Update Checklist - -- [ ] Update relevant markdown files -- [ ] Update diagrams if visual changes -- [ ] Test diagram rendering -- [ ] Update this index if new docs added -- [ ] Update README if major changes -- [ ] Verify all links still work - ---- - -## πŸ“ž Getting Help - -- **Documentation issues:** Check this index for the right document -- **Architecture questions:** See [ARCHITECTURE.md](./ARCHITECTURE.md) -- **Setup problems:** See [INSTALL.md](../../INSTALL.md) troubleshooting -- **Recipe issues:** See recipe-specific troubleshooting guides -- **Code questions:** Check code docstrings and comments - ---- - -## 🀝 Contributing to Documentation - -1. **For typos/small fixes:** Edit the relevant file directly -2. **For new diagrams:** Add to `diagrams/` and update `DIAGRAMS_SUMMARY.md` -3. **For new sections:** Update relevant docs and this index -4. **For new recipes:** Create recipe docs following finance recipe structure - -**Style Guide:** -- Use clear, concise language -- Include code examples where helpful -- Add diagrams for complex concepts -- Keep this index updated -- Test all commands before documenting - ---- - -## πŸ“„ License - -All documentation is part of the NVFlow project and follows the Apache-2.0 license. - ---- - -**Version:** 1.0 -**Last Updated:** January 21, 2026 -**Maintained by:** NVFlow Team - ---- - -## πŸš€ Next Steps - -1. **New to NVFlow?** β†’ Start with [README.md](../../README.md) -2. **Need quick reference?** β†’ See [ARCHITECTURE_QUICK_REFERENCE.md](./ARCHITECTURE_QUICK_REFERENCE.md) -3. **Want deep understanding?** β†’ Read [ARCHITECTURE.md](./ARCHITECTURE.md) -4. **Visual learner?** β†’ Browse [diagrams/](../diagrams/) -5. **Ready to run?** β†’ Follow [docs/recipes/finance/quick-start.md](../recipes/finance/quick-start.md) - -**Happy learning! πŸŽ‰** diff --git a/docs/architecture/ARCHITECTURE_QUICK_REFERENCE.md b/docs/architecture/ARCHITECTURE_QUICK_REFERENCE.md deleted file mode 100644 index b40b399..0000000 --- a/docs/architecture/ARCHITECTURE_QUICK_REFERENCE.md +++ /dev/null @@ -1,276 +0,0 @@ -# NVFlow - Architecture Quick Reference - -> **One-page overview of NVFlow architecture** -> **For:** Quick onboarding and reference -> **See also:** [ARCHITECTURE.md](./ARCHITECTURE.md) for comprehensive details - ---- - -## πŸ—οΈ System Architecture (3 Layers) - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ USER LAYER: CLI, Python API, Scripts β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ FRAMEWORK LAYER: WorkflowRunner, StageRegistry β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ EXECUTION LAYER: NeMo-Skills, Slurm, Containers β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - ---- - -## πŸ“¦ Core Components - -| Component | Purpose | Key Methods | -|-----------|---------|-------------| -| **BaseStage** | Abstract base for all stages | `execute()`, `validate_config()` | -| **StageRegistry** | Hierarchical stage registry | `register()`, `get()`, `list_*()` | -| **WorkflowRunner** | Orchestrates workflow execution | `run()`, `validate_config()` | -| **Console** | Rich terminal UI | `header()`, `info()`, `success()` | - ---- - -## 🎯 Hierarchical Organization - -``` -Recipe (Domain: finance, healthcare, retail) - ↓ -Workflow (Pipeline: download, sdg, sft, eval, grpo) - ↓ -Stage (Task: generate_answers, training, evaluate) -``` - -**Example Path:** `finance.sft.sft` β†’ `SFTStage` class - ---- - -## πŸ“Š Finance Recipe Pipeline (6 Workflows, 42 Stages) - -``` -1. download-sec (1 stage) - └─ Download SEC filings β†’ ~100GB JSON - -2. template-based-sdg (6 stages) [PRODUCTION] - └─ Seed β†’ Questions β†’ Context β†’ Answers β†’ Filter β†’ ~300K Q&A - -3. document-grounded-sdg (7 stages) [EXPERIMENTAL] - └─ Preprocess β†’ Generate β†’ Evaluate β†’ ~800K Q&A - -4. sft (4 stages + 2 shared) - └─ Transform β†’ Prepare β†’ Split β†’ Train β†’ Checkpoints - -5. eval (2 stages, dynamically expanded) - └─ Prepare β†’ Evaluate checkpoints β†’ Compare β†’ Results - -6. grpo (10 stages: 9 active + 1 optional) - └─ GRPO reinforcement learning workflow -``` - ---- - -## πŸš€ Execution Flow (4 Phases) - -``` -1. INIT: Load config β†’ Resolve inheritance β†’ Extract context -2. VALIDATE: Check registry β†’ Validate stages -3. EXECUTE: For each stage β†’ Submit to Slurm β†’ Track dependencies -4. MONITOR: Check status β†’ View logs β†’ Collect results -``` - ---- - -## πŸ’» Technology Stack - -```yaml -Core: - - Python 3.12+, OmegaConf, Typer, Rich - -Execution: - - NeMo-Skills (SDG & pipelines) - - Slurm (cluster scheduling) - - Enroot (containers) - -Models: - - vLLM, SGLang (inference) - - HuggingFace (model loading) -``` - ---- - -## πŸ—‚οΈ Directory Structure - -``` -nvflow/ -β”œβ”€β”€ core/ # Framework (BaseStage, Registry, Runner) -β”œβ”€β”€ cli/ # CLI interface (nflow commands) -└── recipes/ # Domain-specific implementations - β”œβ”€β”€ finance/ # 42 stages, 6 workflows - β”‚ β”œβ”€β”€ stages/ # Stage implementations - β”‚ β”œβ”€β”€ workflows/ # YAML configs - β”‚ └── prompts/ # Prompt templates - └── example/ # Learning & testing -``` - ---- - -## πŸ”§ Common Commands - -```bash -# List all stages -nflow list-stages --recipe finance - -# Get stage info -nflow stage-info finance.sft.sft - -# Run single stage -nflow run sft --config workflow.yaml - -# Run all stages -nflow run-all --config workflow.yaml - -# Validate config -nflow validate --config workflow.yaml -``` - ---- - -## πŸ“ Creating a New Stage (3 Steps) - -```python -# 1. Create stage file: nvflow/recipes/finance/stages/sdg/my_stage.py -from nvflow.core import BaseStage, StageRegistry - -# 2. Implement with decorator -@StageRegistry.register( - recipe="finance", - workflow="my_workflow", - stage="my_stage" -) -class MyStage(BaseStage): - workflow = "my_workflow" - - def execute(self, config, cluster, expname, run_after=None): - # Your implementation - pass -``` - -```yaml -# 3. Add to workflow YAML -recipe: finance -workflow: - name: my_workflow -pipeline_stages: - - my_stage -stages: - my_stage: - # Your config -``` - ---- - -## 🎨 Design Patterns - -| Pattern | Usage | Example | -|---------|-------|---------| -| **Template Method** | BaseStage defines interface | `execute()` method | -| **Registry** | Stage discovery | `StageRegistry.get()` | -| **Decorator** | Stage registration | `@StageRegistry.register()` | -| **Strategy** | Execution modes | Local vs. Slurm | -| **Dependency Injection** | Config passing | `execute(config, cluster, ...)` | - ---- - -## 🏭 Deployment Topology - -``` -Local Machine Slurm Cluster -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ nflow CLI │───SSH───► β”‚ Login Node β”‚ -β”‚ Config YAML β”‚ β”‚ ↓ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ Slurm Scheduler β”‚ - β”‚ ↓ β”‚ - β”‚ Compute Nodes β”‚ - β”‚ β€’ 8Γ— H100 GPUs β”‚ - β”‚ β€’ Enroot containers β”‚ - β”‚ β€’ Shared storage β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - ---- - -## πŸ“Š Data Flow (Finance Recipe) - -``` -SEC API β†’ Filings (100GB) β†’ SDG (300K Q&A) β†’ -Data Prep β†’ Training β†’ Checkpoints β†’ Evaluation β†’ Results -``` - ---- - -## πŸ”‘ Key Features - -βœ… **Modular**: Reusable stages across workflows -βœ… **Declarative**: YAML-based configuration -βœ… **Scalable**: Local to multi-node clusters -βœ… **Reproducible**: Version-controlled configs -βœ… **Extensible**: Easy to add recipes/stages -βœ… **Built on NeMo**: Leverages NVIDIA ecosystem - ---- - -## πŸ“š Documentation Map - -| Document | Purpose | Audience | -|----------|---------|----------| -| **README.md** | Getting started | All users | -| **INSTALL.md** | Cluster setup | DevOps, ML Engineers | -| **ARCHITECTURE.md** | Deep dive (26 KB) | Architects, Contributors | -| **ARCHITECTURE_QUICK_REFERENCE.md** | This page | Quick reference | -| **DIAGRAMS_SUMMARY.md** | Diagram guide | Visual learners | -| **diagrams/*.mmd** | Visual diagrams | All users | -| **docs/recipes/finance/** | Finance recipe | Data Scientists | - ---- - -## 🎯 Use Case: Finance Recipe - -**Goal:** Generate synthetic financial Q&A data and train reasoning models - -**Input:** SEC filings (10-K, 10-Q, 8-K) - -**Process:** -1. Download filings (S&P 500) -2. Generate 300K Q&A pairs (template-based SDG) -3. Prepare training data -4. Fine-tune Qwen3-14B (256 GPUs) -5. Evaluate on benchmarks - -**Output:** Fine-tuned financial reasoning model + evaluation metrics - -**Scale:** ~100GB data β†’ 300K Q&A β†’ 256 GPU training β†’ Production model - ---- - -## πŸ”— Quick Links - -- **Full Architecture:** [ARCHITECTURE.md](./ARCHITECTURE.md) -- **Diagrams:** [diagrams/](../diagrams/) -- **Finance Recipe:** [docs/recipes/finance/README.md](../recipes/finance/README.md) -- **Quick Start:** [docs/recipes/finance/quick-start.md](../recipes/finance/quick-start.md) -- **NeMo-Skills:** https://github.com/NVIDIA/NeMo-Skills - ---- - -## πŸ’‘ Tips - -1. **Start with example recipe** for learning -2. **Use `nflow list-stages`** to discover stages -3. **Check `nflow stage-info`** for stage details -4. **Validate configs** before running: `nflow validate` -5. **Monitor jobs** with `squeue` and log files -6. **Pre-download models** to avoid GPU time waste - ---- - -**Version:** 1.0 | **Updated:** Jan 21, 2026 | **License:** Apache-2.0 diff --git a/docs/architecture/DIAGRAMS_SUMMARY.md b/docs/architecture/DIAGRAMS_SUMMARY.md deleted file mode 100644 index 04c20e2..0000000 --- a/docs/architecture/DIAGRAMS_SUMMARY.md +++ /dev/null @@ -1,310 +0,0 @@ -# NVFlow - Architecture Diagrams Summary - -> **Created:** January 21, 2026 -> **Purpose:** Quick reference guide for all architectural diagrams - ---- - -## πŸ“¦ What's Included - -A comprehensive set of architectural diagrams and documentation for the NVFlow orchestration framework has been created: - -### πŸ“„ Main Documentation -- **[ARCHITECTURE.md](./ARCHITECTURE.md)** - Complete architectural documentation (26 KB) - - High-level architecture overview - - System components and design patterns - - Hierarchical organization (Recipe β†’ Workflow β†’ Stage) - - Execution flow and dependency management - - Finance recipe detailed architecture - - Deployment topology - - Technology stack - - Data flow diagrams - -### πŸ“Š Mermaid Diagrams (in `diagrams/` folder) - -1. **[architecture-overview.mmd](../diagrams/architecture-overview.mmd)** - High-level system architecture - - Shows all major components and their relationships - - User interfaces β†’ Core framework β†’ Recipes β†’ Infrastructure β†’ Storage - -2. **[finance-pipeline.mmd](../diagrams/finance-pipeline.mmd)** - Finance recipe end-to-end pipeline - - Complete data flow from SEC filings to model evaluation - - All 6 workflows with 42 stages visualized - - Production vs. experimental paths - -3. **[execution-flow.mmd](../diagrams/execution-flow.mmd)** - Runtime execution sequence - - Step-by-step workflow execution - - User command β†’ Config loading β†’ Stage execution β†’ Job submission - - Background Slurm job processing - -4. **[component-architecture.mmd](../diagrams/component-architecture.mmd)** - Class diagram - - Core framework classes and relationships - - BaseStage, StageRegistry, WorkflowRunner - - Concrete stage implementations - - External dependencies - -5. **[deployment-architecture.mmd](../diagrams/deployment-architecture.mmd)** - Infrastructure view - - Local development environment - - Slurm cluster topology - - Compute nodes, storage, containers - - Network connections and data flow - -### πŸ“– Diagram Documentation -- **[diagrams/README.md](../diagrams/README.md)** - Guide for viewing and editing diagrams - - Description of each diagram - - Multiple viewing options (online, VS Code, CLI, GitHub) - - Mermaid syntax reference - - Style guide and contribution guidelines - ---- - -## πŸš€ Quick Start Guide - -### Viewing the Architecture - -**Option 1: Read the comprehensive documentation** -```bash -cat ARCHITECTURE.md -# or open in your favorite markdown viewer -code ARCHITECTURE.md -``` - -**Option 2: View diagrams online** -1. Visit https://mermaid.live/ -2. Open any `.mmd` file from `diagrams/` -3. Copy-paste the content -4. View and export as needed - -**Option 3: Generate PNG images** -```bash -cd diagrams/ - -# Install Mermaid CLI if not already installed -npm install -g @mermaid-js/mermaid-cli - -# Generate all diagrams as PNG -mmdc -i architecture-overview.mmd -o architecture-overview.png -mmdc -i finance-pipeline.mmd -o finance-pipeline.png -mmdc -i execution-flow.mmd -o execution-flow.png -mmdc -i component-architecture.mmd -o component-architecture.png -mmdc -i deployment-architecture.mmd -o deployment-architecture.png -``` - -**Option 4: VS Code with Mermaid Preview** -1. Install "Mermaid Preview" extension -2. Open any `.mmd` file -3. Right-click β†’ "Mermaid: Preview" - ---- - -## 🎯 Which Diagram Should I Use? - -### For Different Audiences - -| Audience | Recommended Diagrams | Purpose | -|----------|---------------------|---------| -| **New Users** | `architecture-overview.mmd` | Get a high-level understanding of the system | -| **Data Scientists** | `finance-pipeline.mmd` | Understand the ML pipeline and data flow | -| **ML Engineers** | `execution-flow.mmd`, `finance-pipeline.mmd` | Learn how to run and debug workflows | -| **Software Engineers** | `component-architecture.mmd` | Understand code structure and extend the framework | -| **DevOps/Infrastructure** | `deployment-architecture.mmd` | Set up cluster and infrastructure | -| **System Architects** | All diagrams + `ARCHITECTURE.md` | Comprehensive system understanding | -| **Contributors** | `component-architecture.mmd`, `ARCHITECTURE.md` | Contribute new stages and recipes | - -### For Different Questions - -| Question | Diagram to Check | -|----------|------------------| -| "What does NVFlow do?" | `architecture-overview.mmd` | -| "How do I build an ML pipeline?" | `finance-pipeline.mmd` | -| "How does stage execution work?" | `execution-flow.mmd` | -| "How do I create a new stage?" | `component-architecture.mmd` | -| "What infrastructure do I need?" | `deployment-architecture.mmd` | -| "How are stages organized?" | `component-architecture.mmd` | -| "How does the finance recipe work?" | `finance-pipeline.mmd` | -| "How does NVFlow integrate with Slurm?" | `deployment-architecture.mmd`, `execution-flow.mmd` | - ---- - -## πŸ“‹ Diagram Details - -### 1. Architecture Overview -``` -Components Shown: -βœ“ User Interfaces (CLI, Python API, Scripts) -βœ“ Core Framework (WorkflowRunner, StageRegistry, BaseStage, Console) -βœ“ Recipe Layer (Finance, Example, Custom recipes) -βœ“ External Dependencies (NeMo-Skills, NeMo-RL, Slurm, Containers) -βœ“ Storage Layer (Data, Models, Outputs) - -Use Case: Understanding system boundaries and component relationships -``` - -### 2. Finance Pipeline -``` -Coverage: -βœ“ Complete 6-workflow pipeline (42 stages total) -βœ“ Data acquisition (SEC filings download) -βœ“ SDG (Template-based & Document-grounded approaches) -βœ“ Data preparation (Transformation, formatting, splitting) -βœ“ Training (Multi-node SFT with Qwen3-14B) -βœ“ Evaluation (Benchmarks and baselines) - -Use Case: Understanding the end-to-end ML pipeline -``` - -### 3. Execution Flow -``` -Phases Covered: -βœ“ Initialization (Config loading, validation) -βœ“ Validation (Registry checks, config validation) -βœ“ Execution (Stage execution loop, job submission) -βœ“ Monitoring (Job status, log viewing) - -Use Case: Debugging and understanding runtime behavior -``` - -### 4. Component Architecture -``` -Classes Documented: -βœ“ BaseStage (abstract base class) -βœ“ StageRegistry (hierarchical registry) -βœ“ WorkflowRunner (orchestrator) -βœ“ Concrete stages (SFT, Generate, Download, Evaluate) -βœ“ CLI (user interface) -βœ“ External dependencies (NeMo-Skills, OmegaConf) - -Use Case: Code navigation and extension -``` - -### 5. Deployment Architecture -``` -Infrastructure Components: -βœ“ Local development machine (NVFlow installation) -βœ“ SSH tunnel (secure connection) -βœ“ Slurm cluster (login node, scheduler, compute nodes) -βœ“ GPU compute nodes (H100 GPUs, containers) -βœ“ Shared storage (Lustre/NFS filesystem) -βœ“ Container runtime (Enroot, .sqsh images) - -Use Case: Cluster setup and deployment planning -``` - ---- - -## 🎨 Diagram Rendering Examples - -### In Markdown (GitHub) -````markdown -```mermaid -graph TB - A[NVFlow] --> B[Recipes] - A --> C[Workflows] - A --> D[Stages] -``` -```` - -### In Python Documentation -```python -""" -Architecture: - Recipe β†’ Workflow β†’ Stage - - See: diagrams/architecture-overview.mmd -""" -``` - -### In Presentations -- Export diagrams to PNG/SVG using `mmdc` CLI -- Import into PowerPoint/Keynote/Google Slides -- High resolution for professional presentations - ---- - -## πŸ“Š Diagram Statistics - -| Metric | Count | -|--------|-------| -| Total Diagrams | 5 | -| Total Documentation Pages | 2 (ARCHITECTURE.md + diagrams/README.md) | -| Components Visualized | 50+ | -| Workflows Documented | 6 | -| Stages Documented | 27 | -| Architecture Layers | 5 | - ---- - -## πŸ”„ Keeping Diagrams Updated - -When updating the codebase: - -1. **Adding a new recipe:** - - Update `architecture-overview.mmd` (Recipe Layer section) - - Consider creating a new pipeline diagram (like `finance-pipeline.mmd`) - -2. **Adding a new stage:** - - Update recipe-specific pipeline diagram - - Update `component-architecture.mmd` if it's a new pattern - -3. **Changing core framework:** - - Update `component-architecture.mmd` - - Update `execution-flow.mmd` if execution logic changes - - Update `ARCHITECTURE.md` with detailed explanations - -4. **Infrastructure changes:** - - Update `deployment-architecture.mmd` - - Update cluster setup documentation - -5. **Major architectural changes:** - - Review and update all diagrams - - Update `ARCHITECTURE.md` comprehensively - ---- - -## πŸ“– Related Documentation - -- **[README.md](../../README.md)** - Main project documentation -- **[INSTALL.md](../../INSTALL.md)** - Installation and setup guide -- **[docs/recipes/finance/README.md](../recipes/finance/README.md)** - Finance recipe documentation -- **[docs/recipes/finance/quick-start.md](../recipes/finance/quick-start.md)** - Quick start guide - ---- - -## 🀝 Contributing - -To contribute to the architecture documentation: - -1. **For diagram updates:** - - Edit the `.mmd` files in `diagrams/` - - Test rendering before committing - - Follow the style guide in `diagrams/README.md` - -2. **For documentation updates:** - - Edit `ARCHITECTURE.md` for comprehensive changes - - Keep diagrams and text synchronized - - Use consistent terminology - -3. **For new diagrams:** - - Create new `.mmd` file in `diagrams/` - - Add description to `diagrams/README.md` - - Update this summary file - ---- - -## πŸ“„ License - -All architecture diagrams and documentation are part of the NVFlow project and follow the Apache-2.0 license. - ---- - -## πŸ™ Acknowledgments - -Built on the NVIDIA NeMo ecosystem: -- [NeMo-Skills](https://github.com/NVIDIA/NeMo-Skills) -- [NeMo-RL](https://github.com/NVIDIA-NeMo/RL) -- [NeMo Framework](https://github.com/NVIDIA/NeMo) - ---- - -**Version:** 1.0 -**Last Updated:** January 21, 2026 -**Maintained by:** NVFlow Team diff --git a/docs/cluster-configuration.md b/docs/cluster-configuration.md index 5ca331b..19636b6 100644 --- a/docs/cluster-configuration.md +++ b/docs/cluster-configuration.md @@ -216,8 +216,10 @@ containers: # Required nemo-skills: /path/to/containers/nemo-skills.sqsh vllm: /path/to/containers/vllm.sqsh + vllm-grpo: /path/to/containers/vllm-grpo.sqsh # GRPO rollouts / judge sglang: /path/to/containers/sglang.sqsh - nemo-rl: /path/to/containers/nemo-rl.sqsh + nemo-rl: /path/to/containers/nemo-rl.sqsh # SFT/GRPO training + nemo-gym: /path/to/containers/nemo-gym.sqsh # CPU Gym-only GRPO stages ``` **Details:** @@ -237,33 +239,39 @@ Maps host file system paths to container paths. ```yaml mounts: - :/hf_models # HuggingFace models - - :/workspace # Your workspace + - :/workspace # Writable data dir (outputs + cache) # Add more mounts as needed: # - /lustre/data:/data ``` **Format:** `:` +> **`/workspace` holds writable data, not source code.** Recipe code and +> checked-in assets (prompts, dataset descriptors, Gym overlays) ship to workers +> via the nemo-run packaged snapshot at `/nemo_run/code` (also the job's working +> directory), so the nvflow repo is **not** mounted. Point `/workspace` at a +> dedicated writable data directory holding `/workspace/outputs/**` (stage +> outputs, checkpoints, SEC cache, eval-datasets) and `/workspace/cache/**` +> (`HF_HOME`) β€” not your repo checkout. On an on-cluster launcher (no +> `ssh_tunnel`), keep the launcher's cwd at the repo root so resume/skip +> detection can map `/workspace/outputs/...` back to the host outputs dir. + **Common mounts:** | Host Path | Container Path | Purpose | |-----------|----------------|---------| -| Your workspace directory | `/workspace` | Code, configs, outputs | +| Writable data directory | `/workspace` | Outputs, checkpoints, caches (code ships via `/nemo_run/code`) | | Shared model storage | `/hf_models` | Pre-trained models | | Root Lustre | `/lustre` | Access entire shared filesystem | | Dataset directory | `/data` | Training/evaluation datasets | -### Do NOT bind-mount NeMo-RL / NeMo-Gym source over the image paths - -The self-sufficient `nvflow-nemo-rl` image (built from [`dockerfiles/Dockerfile.nemo-rl`](../dockerfiles/Dockerfile.nemo-rl)) already contains: +### NeMo-RL / NeMo-Gym: trainer image and Gym source -- NeMo-RL source at `/opt/NeMo-RL` (and `/opt/nemo-rl` lowercase alias) -- NeMo-Gym at `/opt/NeMo-RL/3rdparty/Gym-workspace/Gym` (branch `ude/finance-sec-search-v2`) -- A pre-built `.venv` symlinked across all 6 Gym components +SFT and GRPO `training` run on the `nvflow-nemo-rl` image, built from [`dockerfiles/Dockerfile.nemo-rl`](../dockerfiles/Dockerfile.nemo-rl). It bakes the Gym source and one venv per Gym component, so nothing is resolved at job runtime and **no Gym mount is required**. -GRPO stages call `installation_command: source /opt/NeMo-RL/3rdparty/Gym-workspace/Gym/.venv/bin/activate`. Bind-mounting a host source tree at `/opt/NeMo-RL` or `/opt/NeMo-RL/3rdparty/Gym-workspace/Gym` **shadows the baked `.venv`** and breaks `prepare_data`, `collect_rollouts`, `compute_rewards`, and `training` with `No such file or directory`. +The Gym-only GRPO stages (`prepare_data`, `prefetch_cache`, `collect_rollouts`, `compute_rewards`) run on the CPU-only `nvflow-nemo-gym` image, also with baked venvs (`&gym_install_cpu` in `base.yaml`). -The overlay mounts in `template-slurm.yaml` are commented out for exactly this reason. Only uncomment them if you're deliberately iterating on NeMo-RL / Gym source against a host `.venv` you've built to be ABI-compatible with the image. In that dev-mode case you must also set `NRL_FORCE_REBUILD_VENVS=true` (see [Environment Variables](#environment-variables) below) -- which requires internet, so it can only be used on a connected node. +Do not bind-mount Gym or NeMo-RL source over the image in production β€” it shadows the baked tree and invalidates the container fingerprint, forcing a runtime rebuild. To iterate on Gym source in dev mode, mount your clone at `/opt/nemo-rl/3rdparty/Gym-workspace/Gym` and leave `UV_OFFLINE` unset so the editable install can resolve. See [`docs/development/nemo-rl-gym.md`](development/nemo-rl-gym.md). ### Model-Specific Cluster Configs @@ -272,14 +280,14 @@ Some models require additional cluster-level differences (e.g. different timeout | Cluster Config | Used By | Notes | |----------------|---------|-------| | `my_cluster.yaml` | Qwen3, Gemma3 (dense models) | Default for all standard models | -| `my_cluster_nemotron.yaml` | Nemotron-3-Nano (MoE) | Use only if Nemotron needs different mounts/env -- the self-sufficient `nvflow-nemo-rl` image now handles MoE without a host overlay | +| `my_cluster_nemotron.yaml` | Nemotron-3-Nano (MoE) | Use only if Nemotron needs different mounts/env -- the `nemo-rl` image handles MoE without a NeMo-RL source overlay | **How it works:** - `base.yaml` (SFT workflow) sets `cluster: my_cluster` as the default - A model config can override with `cluster: my_cluster_nemotron` - Keep both configs in sync when making infrastructure changes -> **Note:** Previous versions of this guide recommended a NeMo-RL host overlay (`/path/to/RL:/opt/NeMo-RL`) for Nemotron-3-Nano MoE support. With the self-sufficient `nvflow-nemo-rl` image that overlay is no longer required and would shadow the baked `.venv`. See the [SFT Workflow Guide](recipes/finance/workflows/04-sft.md) for the current setup. +> **Note:** No NeMo-RL or Gym source overlay is mounted by default. Nemotron-3-Nano MoE support needs no host overlay, and both `nemo-rl` and `nemo-gym` ship with Gym baked in. See the [SFT Workflow Guide](recipes/finance/workflows/04-sft.md) for the current setup. --- @@ -345,12 +353,10 @@ env_vars: - HF_HUB_OFFLINE=1 - HF_DATASETS_OFFLINE=1 - TRANSFORMERS_OFFLINE=1 - - UV_OFFLINE=true + # - UV_OFFLINE=true # keep unset to allow runtime uv builds; set only for strict airgap - TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache - TIKTOKEN_RS_CACHE_DIR=/opt/tiktoken_cache - TIKTOKEN_ENCODINGS_BASE=/opt/tiktoken_cache - # NeMo-RL / GRPO dev-mode only (do NOT enable in self-sufficient mode) - # - NRL_FORCE_REBUILD_VENVS=true # API keys (keep secret, don't commit to git!) - HF_TOKEN=hf_... - OPENAI_API_KEY=sk-... @@ -377,18 +383,20 @@ These variables prevent the runtime from making outbound network calls and from | `HF_HUB_OFFLINE` | `1` | Disables HuggingFace Hub network access (model + tokenizer downloads) | | `HF_DATASETS_OFFLINE` | `1` | Disables `datasets` network access | | `TRANSFORMERS_OFFLINE` | `1` | Disables `transformers` network access. `huggingface_hub` treats this as equivalent to `HF_HUB_OFFLINE=1` | -| `UV_OFFLINE` | `true` | Prevents `uv` from resolving / downloading packages or Python interpreters at runtime. Keep this set **always** -- containers ship with frozen venvs | +| `UV_OFFLINE` | *unset* | Global flag; **left unset** so components beyond the baked set can be built on demand (see [trainer image and Gym source](#nemo-rl--nemo-gym-trainer-image-and-gym-source)). All GRPO/SFT venvs are baked, so nothing is built at runtime in practice. eval / SDG / SFT never invoke `uv` | | `TIKTOKEN_CACHE_DIR` | `/opt/tiktoken_cache` | Points `tiktoken` at the cache baked into the images | | `TIKTOKEN_RS_CACHE_DIR` | `/opt/tiktoken_cache` | Points the Rust `tiktoken-rs` client at the cache (used by `openai_harmony`) | | `TIKTOKEN_ENCODINGS_BASE` | `/opt/tiktoken_cache` | Required for `openai_harmony` to load `HARMONY_GPT_OSS` offline | -> **One-time connected-node stages:** A few stages (`download_sec_filings`, `create_seed_data`, eval `prepare_data`, GRPO `prepare_data` with `should_download: true`) need internet on first run to pull benchmark/seed datasets. For those submissions, **temporarily comment out** `HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE`, and `TRANSFORMERS_OFFLINE`. Keep `UV_OFFLINE=true` set in all cases. See [INSTALL.md β†’ One-Time Connected-Node Stages](../INSTALL.md#one-time-connected-node-stages-datasets). +> **One-time connected-node stages:** A few stages (`download_sec_filings`, `create_seed_data`, eval `prepare_data`, GRPO `prepare_data` with `should_download: true`) need internet on first run to pull benchmark/seed datasets. For those submissions, **temporarily comment out** `HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE`, and `TRANSFORMERS_OFFLINE`. See [INSTALL.md β†’ One-Time Connected-Node Stages](../INSTALL.md#one-time-connected-node-stages-datasets). -#### NeMo-RL / GRPO Variables (Dev Mode Only) +#### NeMo-RL / GRPO training venv -| Variable | Value | Purpose | -|----------|-------|---------| -| `NRL_FORCE_REBUILD_VENVS` | `true` | **Dev mode only.** Forces Ray workers to rebuild their virtual environments from the mounted NeMo-RL source tree instead of reusing cached venvs. Requires internet (uses `uv` to resolve packages) -- **do not enable in self-sufficient production**. Only relevant when you've bind-mounted a host NeMo-RL / Gym source clone over `/opt/NeMo-RL` and want Ray workers to pick up the new source | +GRPO `training` runs on the `nemo-rl` image, which bakes Gym and one venv per Gym component. Nothing is built at runtime: NeMo-RL matches `/opt/nemo_rl_container_fingerprint` and reuses the baked venvs. Do not bind-mount Gym or NeMo-RL source over the image -- that shadows the baked tree, invalidates the fingerprint, and forces a rebuild. The Gym-only stages run on the self-contained `nvflow-nemo-gym` image, also with baked venvs. + +`UV_OFFLINE` is left unset so components outside the baked set can still be built on demand. Note the consequence: a fingerprint miss will silently rebuild over the cluster proxy rather than fail, so verify airgap behaviour by checking training logs for venv-build activity, not by the job succeeding. eval / SDG / SFT never invoke `uv`. + +See [trainer image and Gym source](#nemo-rl--nemo-gym-trainer-image-and-gym-source) and [`docs/development/nemo-rl-gym.md`](development/nemo-rl-gym.md). #### API Keys (Secrets) diff --git a/docs/development/nemo-rl-gym.md b/docs/development/nemo-rl-gym.md new file mode 100644 index 0000000..811a202 --- /dev/null +++ b/docs/development/nemo-rl-gym.md @@ -0,0 +1,45 @@ +# NeMo-RL / NeMo-Gym: trainer image & Gym venvs (advanced) + +> Audience: **advanced / dev**. For a normal GRPO run you do **not** need this page β€” follow INSTALL.md and the quick-start. This page explains how GRPO `training` gets NeMo-RL and NeMo-Gym, and how to iterate on Gym source. (SFT `training` runs on the same image but never touches Gym.) + +## How the trainer gets NeMo-RL and Gym + +GRPO `training` runs on `nvflow-nemo-rl`, built from [`dockerfiles/Dockerfile.nemo-rl`](../../dockerfiles/Dockerfile.nemo-rl). The NeMo-RL base supplies Transformer Engine and the prebuilt NeMo-RL / Ray venvs but leaves the Gym venvs unbuilt, because upstream gates that prefetch behind `NEMO_GYM_PREFETCH_CONFIGS`. Our image closes exactly that gap and changes nothing else: + +- The Gym submodule is advanced in place to `GYM_REF` and reinstalled editable into the Gym actor venv. +- One venv is baked **per Gym component** under `/opt/gym_venvs`, by driving `gym env start … +dry_run=true` from that actor venv. Driving it this way is what makes Gym pin each component to the container's own interpreter and Ray version. +- `training.py` sets `env.nemo_gym.skip_venv_if_present = True` (`nvflow/recipes/finance/stages/rl/training.py:157`), so NeMo-RL reuses the baked venvs rather than building. +- The nemo-skills `installation_command` for the trainer is a no-op (`"true"`) β€” no Gym CLI setup is needed inside the trainer container. + +The result is that **nothing resolves at job runtime and no Gym mount is required.** + +The Gym-only stages (`prepare_data`, `prefetch_cache`, `collect_rollouts`, `compute_rewards`) run on the CPU-only `nvflow-nemo-gym` image instead, which bakes the Gym CLI (`/opt/gym-cli-venv`) and its own per-component venvs (`/opt/gym-venvs`). They share the `&gym_install_cpu` command in `nvflow/recipes/finance/workflows/grpo/base.yaml`, which only puts the baked CLI on `PATH` β€” no build, no network. See [`docs/maintainers/containers.md`](../maintainers/containers.md) for both builds. + +### Why two venv directories + +`nvflow-nemo-gym` bakes to `/opt/gym-venvs` (hyphen; `gym_uv_venv_dir` in the workflow YAML); the trainer bakes to `/opt/gym_venvs` (underscore; `NEMO_GYM_VENV_DIR`, inherited from the base). + +Aligning the paths would not make the venvs interchangeable. The images differ in Python (3.12 vs 3.13) and Ray (2.56.1 vs 2.55.1), and both are hard constraints: a Gym server joining the trainer's Ray cluster is version-checked on Ray and on Python down to the patch level, and a venv is bound to its interpreter. Each image bakes where its own runtime looks. + +## Dev iteration on Gym source + +To work against a modified Gym, bind-mount your clone over the trainer's Gym path: + +```yaml +mounts: + - /Gym:/opt/nemo-rl/3rdparty/Gym-workspace/Gym +``` + +This is the one configuration where `uv` resolves at runtime, so it needs `UV_OFFLINE` unset and a reachable pypi mirror. `skip_venv_if_present=True` still applies, so remove the stale venv if you want a rebuild. + +**Do not use this mount in production.** It shadows the baked source and venvs and invalidates the container fingerprint, which turns a fully offline run into one that silently rebuilds over the cluster proxy. + +## Why `UV_OFFLINE` stays unset + +The images need no resolve, so setting it would change nothing in a normal run. It is left unset deliberately, to keep the dev-iteration path above working. + +The trade-off is worth stating: because it is unset, a fingerprint miss **rebuilds instead of failing loudly**. So verify airgap behaviour by checking training logs for venv-build activity, not by the job succeeding. + +## Regression guard + +The `&gym_install_cpu` command and the Gym env-start wiring are covered by `tests/test_grpo_gym_install.py` β€” run `uv run pytest tests/test_grpo_gym_install.py -v` before changing either. diff --git a/docs/development/sdg/document_grounded/ADDING_A_DOMAIN.md b/docs/development/sdg/document_grounded/ADDING_A_DOMAIN.md new file mode 100644 index 0000000..c52aae8 --- /dev/null +++ b/docs/development/sdg/document_grounded/ADDING_A_DOMAIN.md @@ -0,0 +1,714 @@ +# Adding a New Domain to DG-SDG + +> Turn a directory of your own documents into a fine-tuning dataset +> (single `final_result.jsonl` consumed by both SFT and GRPO) by adding a +> new "recipe" to nvflow's Document-Grounded SDG pipeline. +> +> **Audience**: anyone β€” a coworker or an AI agent β€” who can read this doc +> (plus the code it links to), gather the domain-specific info, and build a +> new recipe end-to-end. It should be self-contained enough that handing it +> over is all it takes. The pipeline runs on a Slurm cluster. +> +> **Worked example**: a `legal` recipe with court opinions at +> `/data/legal/cases///.html`. Substitute your own +> domain name wherever you see `legal` / ``. + +The section numbers below mirror the phases in the diagram: + +![DG-SDG: adding a new domain](dgsdg-add-new-domain.png) + +## What you write, in 1 picture + +``` +nvflow/recipes// +β”œβ”€β”€ prompts/ +β”‚ β”œβ”€β”€ document_grounded_generate_questions.yaml Β§2 +β”‚ β”œβ”€β”€ document_grounded_verify_questions.yaml Β§2 +β”‚ β”œβ”€β”€ _qa_template.yaml Β§2 (reused in Β§3) +β”‚ β”œβ”€β”€ evaluate_answers.yaml Β§3 +β”‚ └── genselect_answers.yaml Β§2 (cp from finance verbatim) +β”œβ”€β”€ utils/sdg/ +β”‚ β”œβ”€β”€ _data_preprocess.py Β§1 +β”‚ β”œβ”€β”€ _callbacks.py Β§1 + Β§4 +β”‚ └── _question_prep.py Β§1 +β”‚ └── _postprocess.py Β§5 (thin wrapper) +β”œβ”€β”€ stages/sdg/ Β§5 (register shared generic DG-SDG stages) +β”‚ └── __init__.py Β§5 +β”œβ”€β”€ workflows/sdg/ +β”‚ β”œβ”€β”€ document-grounded-sdg.yaml Β§5 +β”‚ └── document-grounded-sdg-demo.yaml Β§5 +β”œβ”€β”€ __init__.py Β§5 +β”œβ”€β”€ stages/__init__.py Β§5 +β”œβ”€β”€ stages/sdg/__init__.py Β§5 +β”œβ”€β”€ utils/__init__.py Β§5 +β”œβ”€β”€ utils/sdg/__init__.py Β§5 +└── recipe.yaml Β§5 +``` + +--- + +## Β§0 Prerequisites + directory skeleton + +Before you start, verify: + +- nvflow repo checked out on the launcher/host; run commands from the repo root (`ls nvflow/recipes/finance` works). At runtime this code ships to workers via the nemo-run packaged snapshot (`/nemo_run/code`) β€” it is not mounted. +- Cluster config exists (`ls cluster_configs/my_cluster.yaml`) +- `nemo-gym` container available (`enroot list | grep nemo-gym`) β€” the gym-only client the shared DG-SDG generation stages run in (Gym source at `/opt/Gym`, per-component venvs baked at `/opt/gym-venvs`; no `nemo-rl` image or runtime `uv sync` needed for SDG) +- Model weights mounted (`ls /hf_models/openai/gpt-oss-120b` and `ls /hf_models/Qwen/Qwen3-235B-A22B-Instruct-2507`) +- `nflow --help` works +- Your raw documents are in one root directory +- A short snake_case domain name picked (this guide uses `legal`) + +Then create the skeleton: + +```bash +cd # repo root +export DOMAIN=legal # CHANGE ME + +mkdir -p nvflow/recipes/$DOMAIN/{prompts,utils/sdg,stages/sdg,workflows/sdg} +touch nvflow/recipes/$DOMAIN/__init__.py \ + nvflow/recipes/$DOMAIN/stages/__init__.py \ + nvflow/recipes/$DOMAIN/stages/sdg/__init__.py \ + nvflow/recipes/$DOMAIN/utils/__init__.py \ + nvflow/recipes/$DOMAIN/utils/sdg/__init__.py +``` + +--- + +## Β§1 Phase 1 Β· PREPARE INPUT + +Three Python files under `utils/sdg/`. They cover the diagram's Phase 1: +turn raw documents into JSONL records, then attach a `context` string to +each record so the LLM has something to read. + +### 1.1 `_data_preprocess.py` + +Walks your raw document tree and writes one JSONL file with one record per +chunk. **You** run this once manually (and the workflow re-runs it as +step 0). The fields you emit here become the input contract for +`context_builder` in 1.2. + +```python +#!/usr/bin/env python3 +"""Preprocess documents into per-chunk JSONL records.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any, Iterable + +from bs4 import BeautifulSoup + +try: + import tiktoken + _ENC = tiktoken.get_encoding("cl100k_base") +except ImportError: + _ENC = None + + +def _tokenize(text: str) -> list[str]: + return _ENC.encode(text) if _ENC is not None else text.split() + + +def _detokenize(tokens) -> str: + return _ENC.decode(tokens) if _ENC is not None else " ".join(tokens) + + +def chunk_text(text: str, max_tokens: int = 2000, overlap_tokens: int = 100) -> Iterable[str]: + toks = _tokenize(text) + step = max(max_tokens - overlap_tokens, 1) + for i in range(0, len(toks), step): + chunk = toks[i : i + max_tokens] + if chunk: + yield _detokenize(chunk) + if i + max_tokens >= len(toks): + break + + +def extract_metadata(html_path: Path) -> dict[str, Any]: + # CUSTOMIZE for your file layout. The keys returned here must be a + # superset of what context_builder reads in 1.2. + parts = html_path.parts + try: + court, year = parts[-3], parts[-2] + except IndexError: + court, year = "", "" + return { + "case_name": re.sub(r"[_\-]+", " ", html_path.stem).strip(), + "court": court, + "decision_year": year, + "section": "Opinion", + "doc_path": str(html_path), + } + + +def extract_body_text(html_path: Path) -> str: + soup = BeautifulSoup(html_path.read_text(encoding="utf-8", errors="ignore"), "html.parser") + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + return soup.get_text(separator="\n", strip=True) + + +def main(input_dir: Path, output_dir: Path, max_tokens: int, overlap_tokens: int) -> int: + output_dir.mkdir(parents=True, exist_ok=True) + out_path = output_dir / f"{input_dir.name}-data.jsonl" + n_records = 0 + with open(out_path, "w", encoding="utf-8") as out: + for html_path in sorted(input_dir.rglob("*.html")): + meta = extract_metadata(html_path) + body = extract_body_text(html_path) + if not body.strip(): + continue + for chunk_idx, chunk_str in enumerate(chunk_text(body, max_tokens, overlap_tokens)): + rec = {**meta, "chunk_id": chunk_idx, "content": chunk_str} + out.write(json.dumps(rec, ensure_ascii=False) + "\n") + n_records += 1 + print(f"Wrote {n_records} records to {out_path}") + return n_records + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--input_dir", type=Path, required=True) + p.add_argument("--output_dir", type=Path, required=True) + p.add_argument("--max_tokens", type=int, default=2000) + p.add_argument("--overlap_tokens", type=int, default=100) + # The Stage 0 shim (generic_stage/sdg/document_grounded/dg_sdg_preprocess.py) ALWAYS + # passes these four extra flags too. You must accept them even if your + # domain doesn't sample by a distribution -- otherwise argparse aborts + # the Slurm job with "unrecognized arguments". Ignore the ones you don't + # use (finance reads multiple CSVs from --distribution_dir; see note below). + p.add_argument("--distribution_dir", type=Path, default=None) + p.add_argument("--total_samples", type=int, default=150000) + p.add_argument("--max_skip_count", type=int, default=20000) + p.add_argument("--seed", type=int, default=42) + args = p.parse_args() + main(args.input_dir, args.output_dir, args.max_tokens, args.overlap_tokens) +``` + +> **Stage 0 CLI contract β€” accept all 8 flags.** The generic shim invokes +> your module as +> `python3 -m _data_preprocess --input_dir … --output_dir … +> --distribution_dir … --max_tokens … --overlap_tokens … --total_samples … +> --max_skip_count … --seed …`. Your argparse must define every one of these +> (the four above plus the four sampling flags) or the job crashes before it +> does any work. `distribution_dir` is currently **required by the shim's +> `validate_config`**, so the workflow YAML must set +> `stages.dg_sdg_preprocess.distribution_dir` even if your CLI ignores it +> (point it at a dir with a placeholder CSV). +> +> **Multi-CSV input is supported.** `--distribution_dir` is a *directory*, not +> a single file, so a domain can read any number of CSVs from it. Finance +> reads four (`{10k,10q}_{1company,2company}_distribution.csv`); there is no +> generic constraint on count or naming β€” your CLI decides what to load. +> +> For non-HTML inputs replace `extract_body_text` with whatever extracts +> text from your format (`.read_text()`, `pypdf`, `pdfminer.six`, etc.). +> Chunking + metadata logic stays the same. +> +> **Finance counterpart** (for reference): `nvflow/recipes/finance/utils/sdg/dg_sdg_data_preprocess.py`. +> Keep the `_data_` infix to avoid confusion with the stage name +> `dg_sdg_preprocess`. + +### 1.2 `_callbacks.py` (context_builder) + +A pure function: `(record) β†’ str`. Called by the library once per record +during step-1 question generation. Empty string β‡’ skip the record. + +The fields you reference here MUST match what 1.1 writes. + +```python +"""Domain-specific callbacks for the DG-SDG recipe.""" + +from typing import Any + + +def legal_context_builder(record: dict[str, Any]) -> str: + case_name = record.get("case_name", "") + court = record.get("court", "") + year = record.get("decision_year", "") + section = record.get("section", "Opinion") + content = record.get("content", "") + + if not content: + return "" + + return ( + f"**{year} {court}: {case_name}**\n\n" + f"**Section: {section}**\n\n" + f"{content}\n" + ) + +# Β§4 will add is_legal_sft_eligible / is_legal_rl_eligible to this same file. +``` + +> **Finance counterpart**: `nvflow/recipes/finance/utils/sdg/sec_callbacks.py` +> (finance uses `sec_*` prefix not `finance_*`; the file is named after the +> SECQUE benchmark for historical reasons). + +### 1.3 `_question_prep.py` + +A 20-line CLI that bolts `context_builder` into the lib's generic helper. +This is the only place `context_builder` is actually invoked, and it's +what the workflow's step-1 entrypoint calls. + +```python +"""Thin CLI: construct_question_generate_input with the legal context_builder.""" + +import argparse +from pathlib import Path + +from nvflow.lib.sdg.document_grounded.preprocess import construct_question_generate_input +from nvflow.recipes.legal.utils.sdg.legal_callbacks import legal_context_builder + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--input_folder", type=Path, required=True) + p.add_argument("--output_file", type=Path, required=True) + args = p.parse_args() + + construct_question_generate_input( + args.input_folder, + args.output_file, + context_builder=legal_context_builder, + ) +``` + +> **Finance counterpart**: `nvflow/recipes/finance/utils/sdg/sec_question_prep.py`. + +--- + +## Β§2 Phase 2 Β· GENERATE Q&A + +Four prompt YAMLs under `prompts/`. Two are domain-specific (you write +them), two come straight from finance (copy verbatim). + +> **JSON braces in YAML prompts**: literal `{` / `}` must be doubled +> (`{{` / `}}`) because Python `.format()` substitutes `{context}` etc. + +### 2.1 `document_grounded_generate_questions.yaml` (Q-gen) + +Generates ~12 questions per chunk in valid JSON the lib can parse. + +```yaml +# nvflow/recipes/legal/prompts/document_grounded_generate_questions.yaml +user: |- + You are a senior legal analyst. You will be given an excerpt from a court + opinion or other legal document. + + Your task is to propose the most important questions a legal researcher + should ask about the excerpt. Generate exactly 3 questions for EACH of + the following categories: + - Holding_and_Reasoning + - Procedural_History + - Legal_Standard_Applied + - Implications_and_Precedent + + Respond ONLY with a valid JSON object. No markdown, no commentary. + + Output Format: + {{ + "Holding_and_Reasoning": ["q1", "q2", "q3"], + "Procedural_History": ["q1", "q2", "q3"], + "Legal_Standard_Applied": ["q1", "q2", "q3"], + "Implications_and_Precedent": ["q1", "q2", "q3"] + }} + + Document: {context} +``` + +> If you change the schema (different categories / counts), you also need a +> custom `generation_parser` for the next stage β€” easier to keep this shape. + +### 2.2 `document_grounded_verify_questions.yaml` (Q-verify) + +Per-question Yes/No verdict. **The `system:` block is mandatory** β€” +without "respond ONLY Yes/No" the verifier regex silently drops ~30%+ of +valid questions. + +```yaml +# nvflow/recipes/legal/prompts/document_grounded_verify_questions.yaml +system: |- + You are a legal expert validating analytical questions. Decide if a given + question is valid, expert-level, and answerable using only the Reference + Text. + + Criteria for "Yes": + 1. The Reference Text contains the facts needed to answer. + 2. The question is non-trivial and assesses legal reasoning. + + Criteria for "No": + 1. The Reference Text lacks the specific data or context. + 2. The question is malformed or unrelated. + + Respond ONLY with "Yes" or "No". + +user: |- + **Reference Text:** + {context} + + **Question:** + {problem} + + Is this a valid expert-level question answerable from the text? +``` + +### 2.3 `_qa_template.yaml` (A-gen prompt) + +The single-turn answer prompt used by the answer-generation stage +(`generate_answers`, step-2). + +```yaml +# nvflow/recipes/legal/prompts/legal_qa_template.yaml +user: |- + You are a legal expert. Given a court-opinion excerpt and a question + written by a senior analyst, answer using ONLY the provided text. Do not + use external knowledge. Be concise but precise. If the text does not + support an answer, say so explicitly. + + Document: {context} + + Question: {problem} + + Answer: +``` + +> **Finance counterpart**: `nvflow/recipes/finance/prompts/secque_template.yaml` +> (the prod workflow YAML references it once, as the `generate_answers` stage's +> `++prompt_config`). + +### 2.4 `genselect_answers.yaml` (best-of-N picker) + +Generic, copy verbatim from finance: + +```bash +cp nvflow/recipes/finance/prompts/genselect_answers.yaml \ + nvflow/recipes/$DOMAIN/prompts/genselect_answers.yaml +``` + +--- + +## Β§3 Phase 3 Β· REFINE + +One prompt YAML you write: the judge for the `evaluate_answers` stage. +(`aggregate_answers` then folds the per-seed verdicts into a consensus +`answerable` and needs no prompt.) + +### 3.1 `evaluate_answers.yaml` (judge for seed-evaluation stage) + +Must emit a one-line JSON tag `{"answerable": "YES/NO", "correct": "YES/NO"}` +on the **last** line β€” `evaluate.parse_evaluation` regex looks for exactly +that shape. + +```yaml +# nvflow/recipes/legal/prompts/evaluate_answers.yaml +user: |- + You are evaluating an AI assistant's answer to a legal question grounded + in the provided court-opinion excerpt. + + You need to decide TWO things: + 1. ANSWERABLE: can the question be answered using only the excerpt? + 2. CORRECT: is the assistant's response appropriate? + + ANSWERABLE assessment: + - YES: excerpt contains the necessary facts / citations / reasoning. + - NO: excerpt lacks the necessary information. + + CORRECT assessment: + - When ANSWERABLE=YES: assistant gives an accurate, well-supported answer. + - When ANSWERABLE=NO: assistant correctly identifies info is missing. + + Provide your reasoning first, then end with this exact JSON tag on a new line: + {{"answerable": "YES/NO", "correct": "YES/NO"}} + + Document: {context} + + Question: {problem} + + Assistant's Answer: {generation} +``` + +--- + +## Β§4 Phase 4 Β· SHIP + +No subset-eligibility callbacks are needed. The pipeline emits a single +`final_result.jsonl` per run; downstream SFT / GRPO workflows pick +records by reading that file directly. If you later need a curated SFT +or GRPO subset, do it as a separate post-process step outside DG-SDG +(e.g. a small CLI in your recipe's `utils/`). + +The only domain-specific callback used by the shared stages is the +context-builder (`_context_builder`) wired into +`generate_verified_questions` via the `question_prep_script`, which you +already added in Β§1.2. + +--- + +## Β§5 Workflow wiring + launch + +Last lap: register shared DG-SDG stages, add one postprocess wrapper, +drop in 5 registration files, write the 2 workflow YAMLs, then launch. + +### 5.1 Register shared DG-SDG stages + add postprocess wrapper + +Add a thin domain wrapper around `nvflow.lib.sdg.document_grounded.postprocess`. +There is nothing domain-specific to inject by default β€” it exists only +so the workflow YAML can point at a recipe-owned path, leaving room for +domain-specific cleaning later: + +```python +# nvflow/recipes/legal/utils/sdg/legal_postprocess.py +import argparse +import os +import sys + +from nvflow.lib.sdg.document_grounded.postprocess import dgsdg_post_process + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Post-process DG-SDG data for the legal recipe." + ) + parser.add_argument("--input_file", required=True) + parser.add_argument("--output_dir", required=True) + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + if not os.path.exists(args.input_file): + sys.exit(f"Input file not found: {args.input_file}") + + dgsdg_post_process(args.input_file, args.output_dir, seed=args.seed) +``` + +### 5.2 `__init__.py` Γ— 3 + `recipe.yaml` + +```python +# nvflow/recipes/legal/__init__.py +from . import stages # noqa: F401 +``` + +```python +# nvflow/recipes/legal/stages/__init__.py +from . import sdg # noqa: F401 +``` + +```python +# nvflow/recipes/legal/stages/sdg/__init__.py +from nvflow.generic_stage.sdg.document_grounded import register_for_recipe + +register_for_recipe("legal") +``` + +```yaml +# nvflow/recipes/legal/recipe.yaml +recipe: legal +description: "End-to-end pipeline for legal-domain model training and evaluation" + +workflow_order: + - document_grounded_sdg +``` + +### 5.3 Production workflow YAML + +Start from finance and edit paths: + +```bash +cp nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml \ + nvflow/recipes/$DOMAIN/workflows/sdg/document-grounded-sdg.yaml +``` + +Then edit (search for the strings on the left): + +| Find | Replace with | +| ------------------------------------------------------------- | ------------------------------------------------------------------------------------- | +| `recipe: finance` | `recipe: legal` | +| `description: "generate synthetic finance data ..."` | `description: "generate synthetic legal data ..."` | +| `base_data_dir: /workspace/outputs/finance/...` | `base_data_dir: /workspace/outputs/legal/workflow-document-grounded-sdg` | +| `filings_dir: /workspace/outputs/finance/...` | `filings_dir: /data/legal/cases` (your raw doc root) | +| `pipeline_stages: [- dg_sdg_preprocess, ...]` | Keep as-is. Order is `dg_sdg_preprocess β†’ generate_verified_questions β†’ generate_answers β†’ gym_genselect_answers β†’ evaluate_answers β†’ aggregate_answers β†’ dgsdg_post_process` (7 shared stages) | +| `stages: dg_sdg_preprocess:` block name | Keep block name; set `preprocess_module` to your `_data_preprocess` module path | +| `stages: generate_verified_questions:` block | Set `question_prep_script: nvflow/recipes//utils/sdg/_question_prep.py` | +| `stages: generate_answers:` block | Nothing domain-specific; inherits `gym_*` from prod. Tune `answer_preprocess_kwargs.threshold` if needed | +| `stages: gym_genselect_answers:` block | Set `prompt_template: nvflow/recipes//prompts/genselect_answers.yaml` | +| `stages: dgsdg_post_process:` block | Add `postprocess_script: nvflow/recipes//utils/sdg/_postprocess.py` | +| `++prompt_config=…/prompts/document_grounded_*.yaml` (Q-gen + Q-verify) | Repoint both to `nvflow/recipes/legal/prompts/…` | +| `++prompt_config=…/prompts/secque_template.yaml` (A-gen) | `…/prompts/legal_qa_template.yaml` | +| `prompt_template: …/prompts/evaluate_answers.yaml` (evaluate stage) | `nvflow/recipes/legal/prompts/evaluate_answers.yaml` | +| `finance_domain_keep_fields:` anchor + 6 stage refs | Rename anchor to `_domain_keep_fields:`, replace member fields with **every** domain key your callbacks / preprocess CLI write to JSONL that you want to survive to final training data. (Stages 1–6 carry the anchor; Stage 0 `dg_sdg_preprocess` does not trim.) See Β§5.6 for the full mechanics. | + +> **Don't touch** `gym_path`, `gym_container`, +> `gym_config_paths_format_verification`, `gym_agent_format_verification`, +> `verifier_passthrough`, `verifier_parse_vote` β€” they reference the gym-only +> container, upstream Gym envs, and SDG overlay YAMLs, all of which are +> domain-agnostic. +> +> **Don't touch** model paths under `args: model: /hf_models/…` unless you +> want different models. Finance defaults (gpt-oss-120b for Q-gen + A-gen, +> Qwen3-235B for Q-verify + judges) are strong general-purpose choices. + +### 5.4 Demo workflow YAML + +Inherit prod via `_base_:` and override just the "make it small + fast" +knobs: + +```yaml +# nvflow/recipes/legal/workflows/sdg/document-grounded-sdg-demo.yaml +recipe: legal +workflow: + name: "document_grounded_sdg" + type: "sdg" + description: "demo (smoke test) of document-grounded SDG for legal" + +cluster: my_cluster +_base_: document-grounded-sdg.yaml + +base_data_dir: /workspace/outputs/legal/demo/workflow-document-grounded-sdg-demo +filings_dir: /data/legal/cases # or a small subdir for smoke + +stages: + dg_sdg_preprocess: + max_tokens: 2000 + overlap_tokens: 200 + total_samples: 200 # prod uses 150_000 + + generate_verified_questions: + question_verify_kwargs: + args: + num_random_seeds: 3 + num_chunks: 4 + + generate_answers: + answer_preprocess_kwargs: + threshold: 0.5 + answer_generation_kwargs: + args: + num_random_seeds: 3 + + gym_genselect_answers: + num_chunks: 1 + num_random_seeds: 1 + + evaluate_answers: + num_chunks: 1 + num_random_seeds: 1 +``` + +> Domain paths (`preprocess_module`, `question_prep_script`, `postprocess_script`, +> `prompt_template`) are inherited from prod via `_base_:` deep-merge β€” no need +> to repeat them in demo. + +### 5.5 Launch + +```bash +uv run nflow run-all \ + --config nvflow/recipes/$DOMAIN/workflows/sdg/document-grounded-sdg-demo.yaml +``` + +That submits all 7 stages with `afterok` Slurm dependencies and returns +immediately. + +Output lands in `base_data_dir`: + +``` +$base_data_dir/ +β”œβ”€β”€ step-0-preprocess/jsonl/*.jsonl +β”œβ”€β”€ step-1-questions/ +β”‚ β”œβ”€β”€ generated/ # raw Q-gen rollouts +β”‚ └── verified/ # Q-verify rollouts (consumed by step-2) +β”œβ”€β”€ step-2-answers/ +β”‚ └── generated/output-rs*.jsonl # N candidate answers per question +β”œβ”€β”€ step-3-genselect/selected_answers.jsonl +β”œβ”€β”€ step-4-evaluate/output-rs*.jsonl +β”œβ”€β”€ step-5-aggregate/aggregated_answers.jsonl +└── step-6-post-process/ + └── final_result.jsonl # single cleaned + renamed dataset for SFT / GRPO +``` + +To launch production (after demo works): swap to +`document-grounded-sdg.yaml` (no `-demo` suffix). + +### 5.6 Stage boundary trim (`domain_keep_fields`) + +Every generic DG-SDG stage (Stages 1 through 6) projects its output JSONL +to an allowlist before the next stage reads it. The trim runs inside the +same Slurm job that produces the output, so there is **no extra dependency +to wire and no extra wall-time cost**. + +The allowlist is the set union: + +``` +STAGE_KEEP[stage] # generic fields the lib code produces / needs +| domain_keep_fields # extra fields your recipe writes that you want to survive +- ALWAYS_DROP # NeMo-Gym noise that we always strip +``` + +`STAGE_KEEP[stage]` and `ALWAYS_DROP` live in +[`nvflow/generic_stage/sdg/document_grounded/_schemas.py`](../../../../nvflow/generic_stage/sdg/document_grounded/_schemas.py) β€” +you should not need to edit either when adding a new domain. + +**What you write**: one YAML anchor in your prod workflow YAML and a +reference from each of the 6 trim-eligible stage blocks (Stage 0 +`dg_sdg_preprocess` does not trim because it manufactures the initial +JSONL from raw documents): + +```yaml +# ---- top of document-grounded-sdg.yaml ---- +legal_domain_keep_fields: &legal_domain_keep_fields + - case_id # every field your callbacks / preprocess CLI + - jurisdiction # write into JSONL that you want to survive + - filing_year # all the way to step-6-post-process + # ... (omit raw text fields like `content*` β€” see gotcha below) + +stages: + generate_verified_questions: + # ... existing keys ... + domain_keep_fields: *legal_domain_keep_fields + generate_answers: + domain_keep_fields: *legal_domain_keep_fields + gym_genselect_answers: + domain_keep_fields: *legal_domain_keep_fields + evaluate_answers: + domain_keep_fields: *legal_domain_keep_fields + aggregate_answers: + domain_keep_fields: *legal_domain_keep_fields + dgsdg_post_process: + domain_keep_fields: *legal_domain_keep_fields +``` + +The demo YAML inherits everything via `_base_:` deep-merge β€” no override +needed. + +**Generic `STAGE_KEEP` cheat-sheet** (for context β€” you don't need to +list these in `domain_keep_fields`): + +| Stage | Keep | +| ----------------------------- | ---- | +| `generate_verified_questions` | `context`, `problem`, `question_type`, `generation` | +| `generate_answers` | + `question_voting_pass_rate`, `question_voting_total`, `reasoning_content`, and the Responses-API original form of each candidate answer (`answer_response`, `answer_responses_create_params`) | +| `gym_genselect_answers` | + `reference_answer`, `reference_reasoning`, the picked answer's Responses-API original form (`reference_response`, `reference_responses_create_params`), `genselect_answers_metadata` (drops multi-candidate scaffolding) | +| `evaluate_answers` | as above + `evaluate_generation` | +| `aggregate_answers` | as above + `answerable` (drops `evaluate_generation`) | +| `dgsdg_post_process` | renames `reference_*` β†’ `answer` / `reasoning_content` / `response` / `responses_create_params`, adds `expected_answer` (mirrors `answer`), drops `generation` + `genselect_answers_metadata`; keeps voting stats | + +**Common gotchas:** + +- **Silent drop**: if your `context_builder` or postprocess wrapper writes + a field that's not in `domain_keep_fields`, it is **silently removed at + the first stage boundary**. The final training data won't have it. Add + the field to the anchor. +- **`content0/1/2` / raw text**: finance intentionally omits these. The + Q-prep callback folds them into `context`, so the raw markdown is + redundant after Stage 0. If your domain produces raw text that you want + to ship to SFT, either fold it into `context` in your callback or list + it in `domain_keep_fields`. +- **Per-record schema variance**: if your domain emits records with + different field shapes (finance has 1-company vs 2-company variants), + list the **union** of all variants in the anchor. The trim allowlist + treats missing fields as a no-op (no error). +- **Stage 0 has no trim**: it writes whatever your `_data_preprocess` + CLI writes. If you write junk fields, Stage 1's trim catches them, but + it's cleaner to write only the fields you intend to propagate. diff --git a/docs/development/sdg/document_grounded/dgsdg-add-new-domain.png b/docs/development/sdg/document_grounded/dgsdg-add-new-domain.png new file mode 100644 index 0000000000000000000000000000000000000000..1bbf779e82d06db25d3529d1e163f78f455f193c GIT binary patch literal 298475 zcmd>m2V7Ih_CG!JfPi!eAjLxOy*EX`iYy2O2oNa=2}qF^Kzg-;bkPMD1yL6jsZv#% zfTE)GCLmRj&i~#du;{w&z1`>Uy}uv4xp(f&+$pDg&zUn5ZEkAFK+8o-LPEk|Y@~OX zgoJ_#`0qze1*CXoF4&Wh9KP+Pf5OY%*9qx{AQ6_;CEN+i>I+M03(H7>f3mucNVqcw z?hZU49ff7}gk@wLow1J2;&4YtaSw!#xFgCPj`RSMfgB!icLX7io}u^=JwqVr>ugd^ zZYUohUF1i{oyvitV4?Y=FD3AqI|iIf!yZxB*2ReF7*feQkIcIOABLYjTj8M`Gg>v&kqQ6XZKzVo|9K62B2*+Sh zK3`^YLb(yghxk=A+nGNxvS)&C$f5P(Xd#W|;6|`xPVvU4$Dr*qZ}M{Eo1U!Cx=CH5?d( z2eB*vPIX>LFJJ`mGk+TH=KUqT$%1qXkQ~y(8N31h10)Juw-YJW%WrcEyb!)Xi*&GF z7?dkQ7v+Y+0AuKZ@&GE;0hq$=t0WgMcQ+t|H1HPghIIA-?j3+WA%Idkrx6%0Brv(! z#24;JM@R58I!-7LFCy^)gG_uTHUX?@lQw=C9pZq2H6k{N9nAInv$i?G0J|Vi?g%f8 zAJA`N@h!?BvI>|NWF4Ph=uT;q>|C~yoU9xnADl?)&RaEurT+{y0C|YS^!u&<2F+o; z?Y}+OM_qvVmqH`a2sb1!RWee55CN`0TM6<6D*@aAfBJ9-m(4Wb7wZDDEFgKL0Z}96 zg|EaB?gbJFD3b^z#;OD2Bu<=USJ!% zP{0N4<_*#v(gSD#I6~i#)TBYu!jm46)By6qKW~z>u#Buc{@;Iy1eJ&+N2EY``Cn0> zw3Hk%&vp_7#}ORL??QrZ2xkOH0bm2R2^(PpDPd&;@J}0jBP^rf28t1T;Kmue@k4n7 z$wbO9a6q9!R`hZ~f>Z-i&t?IDIkw8+>yZ-Hl@ZoaB-|qa0pJIe!izG58y2K8F$WiK z4_ADWF7SBT#N?o`40vaXIEWHA^aIp1_;ke08v|-=fQ8(VSS%6+=w~q9-5XGE;#dR% zq-wacGX~)d_u6_ZtS=)BOp~_4FZ|(#w8y~lk_7ICLBIi33(8WE{K1ZYNAlj3q}#|@ z0srs&kh7xFpCxA^iGLSzMsI4CU-ag$CK3>T9*zKg5Y!wX^Sls8&~OJ(&H4Z^3>e;R zT5!9L1O$qclaq{t1CZrQg$lfJRIpc&SJ-;9iFZJuL}&pT{!i*Z=}m=4ls;R}+Z3R( z!Z)PP7RdZg^CkXm4Y(;lwxox-k@gXNfFpqk272*@7%@MrZ?1h9d}(ZIe(dOG(Jy6> z0^A*(S`T|H_ygqmqi4Lp1N93~uLBV{0mXo+mQ}{rAgiPV@Wi$Rf@s8gA<*Jd;%E#4 zjX^meu%Jl9{{bHbZ}9J>#XY?dSWqy5uf;s^*#V(M5VJtb2_GiZ_$vzj_d%T-!U+`P ze+JS?%K{m(0D}NB*#uuqPx{+by~TWg1FZWavXv#0^7o=zf{pjJ)+El~?^2DN9022f zMK#-CnlK3Qe}fSIh-!gR`ASn`-~{Wy!P^}G@&L7M)x5=8K-yQ#Aca?y3c|`n<^R>| zuReiBV8roS8zANVN6bu3fL{#40flk=M?Sat31JyLM=9z6<}02f6amv6KpzOeO8sUp z|BOQ4F3Eo_`yHJ?t-MV={7Xg>9=ZNnDg2%6vuRWTT?TtiR0^`chNQ}}04ai5wj z1UGxQgDdtwKlntcVQcS=baO;t#0U}#Gz_+n6M*x!C}|5j5OM$(5+KQ?oKP6RTLr3d z;ymId5crJrz?;kjJ6!lHIYmgtTQ^dGxw2hS{)gva6N%ti;XgkQvf@M^hpve+Flb&b zaFEhn0KdexG5uIx$E`Ua#TCLKiq5LHj<7ucnP^72Bt6Vtwj;!ymPibIK* z@B5%KV0ytB-mWt634ha+|JT@lvYX)ek3h7v^7oyuziRt|vkU%hb0Hb(o9Z9dJ_;It zhqeSY0j_RG(Z7Rrf2Ga833h*1yZ??M_@$wTZ~t~XP(~c?f%QRPKvDOrm$Kr3Zw`xa z1D9rg0VcmI(*CbnhMTPUKW`bzi2v7EhKE4%+D2I0ctlFfl)z91ctH6zDB3KvRpO91 z;7LF_{RVypYTNY4T7p2wz!cQor%_&rKde~$kJ{(}-~=Q9gb~<3tjiGLapY@e+Ufz& zATfKuBO{J-5(oQ^Mmo3>S8Tt)4Buq1?`bstjhyx0Y%~IVM_9D_8Ug%)#RB-~+|eL< zHvsG^yoH`im^yMm~Ro8u_-7xMdv@f)J!O8v_t2 z*1-kg4zBKrAtan7fM3uaaR6HEjKU!CD|o`Xvck&XYKtuJf3qa0Y>)uw#T0=ODL3R3oNCc7sj~HIe{}YtB1)yw+IO}_i4sm%D-~oGJEfQR{ zbONLf_=EI-djJbh0J#wsCh-mh4^TU9I;^DJ;2zH2V4MxevsfUW=Qjxqlze}asq=rz zy8XvB+@G~>r8ncMzK?kei14qK+y9s5t-QiFyb=Egmo{bps(B0QXz*{Fd3#vjz}QrO z8_WL{#PPQpvU1`G(6It27-B0yiM6#tM3Cw7;+qi{e_Bd?6Hou8__Tj!DRYzA{^t!= z+5aAcbsGWV?Nczy2Q>S(@as0__`-?5nPuz!x5#wsB|zU|ju^lP3#e1z3*wGQEF6d< zLV)Y1KpJsn6Zq~|3iq3&{5?&>zmbOjn@vN2!v9UvaFg)9Cx|@d?T80~TUdHKWF(q4 z2zM}~^J_~~3RrsDN+p^z1T+k{$4BZB^aSw@!5aEHzoq^W&~Ib*bvhyF?(0-w=^x;d zUz+l-RShoZ|Ee!woF5RliUqDHAMl>Ac(f$~e%CwtKeOQTk8}2)g~Z=?!AI_&M#Kn( z-w5_oRJI4A$v0R3zIU*n;)(B3wVPALH}QSJb;~J z+Tb=}ATrIt4d5pvxHD#3bo4JI@N29L?kYg|BE3!kg{36qiBFcm6S%Jre4_`-Qm`QY z0SsZrKX?H9KDQMDAHh;v?|vx;zS7$)M`-*vHsN4_=v(~eA0WqO2(}j-;|wT2{BC13 zvEK;L{>G=^_Za>DW={fa1-E=|+p3B|0PAx|VD<5rT7X5e@3HX|E<>~+x2+fb?@cCg z;=Y(nV!!{VCUY}f|9^Tii4*t5WD+Iw|I%clFyIbsfO4h&sW4WAw3ECOh#h_vxhy54 zq~r)nU&5R1UJXY$)@5tU*3CmR zzKdW5r|-`?IV{W$A22X6vj942VSL!w6gW#{3t9aRM*Nkn_Z65U)bW?%Ri&_-M~7@} zQ2~a2>mInpZyeu?GM(_>0dE+-%`f58~-4 zJ}8VU;dZOj|BarLZT$K-5dR+$_hw-C_oMB9%|hIqY=9bw+p{+9C-Kc_J8{se`VQfB zpdkekDZz{aL*=kuK)4N%1Rf^}@iT(E8sLfdD%B zUkr8pd%?sOlKuin|Jgm@(lXz3Z39E2?4g^STrC2odCxbo>9TWMZjoXa68Ml zV|4+h-Be7Qf)~F{0IcCF5ey`J#RG&kfqmN4$-phXRT6}*ZnNP3{guMMbZ{FG+WLp5 zX?@S*hoq(DejWAs8|P^O69kU&KM9g$0kLlI8wU;I-+ghU<90I`|911lNx};WsNGtIuN$XO)Upq4-uU zxY*c`{p^Pzp{H@1`gd1=5bJDjvv zzFCQZR>LJhCUb~ROo{<-_!nOw)&V8Gc$1om)EmY`DNJZ5?m#?AaB#czxWUirn=S97 z)z^x=MZJrLEFk2-4(&a9gpq^UGx@Tszn{|4giSk`)0vBy=L>1ls)ibboesOIO{cY{ z)fKh_V|{kNX6W&d7F{^H>N;7oW=LXI9lrf^_ljw znL0h)*;(4$+CHIg6Q~-k>HPUSnJz$Ub7@TzLwAOb#YMHEsE6IO%1>v{!OmRJ9-?9D zKEsqVA64tAv~sy~HcwziSrTEu(dEg`u`k%T)^Di#eURk4Q1iq_ohtVh!Gg%C@8$lRX5~l1Uc7i%8luU~W22t^p2+G%C5tn^+f*3vV7tQjCS9%PPNaT_P!*Yj2$fuB#fW98%H5IO0VzH>eqvl{QR29CF&)Igd4Xk>b&MH#msYMXhJ2 zxqR>0fmo(??KajN^((N)C>B@}qY@kG>s$@NmXl-+Nh1?fUSzOSjB_bQQ62ILcWy*Y zQ@(cK7R(CcI2R)@b=vx=XIt_dy$XD{`hX{Ib5#{eb>P}MI1JtXxh9zQ= zVwwypB=+TYMd!0mVESu$c_Nv1oC!KVz*hT?Q6+lhYDjID36nfzV*U)vweiOHQ2Xu+ zJ`}U2FcmqbMMf2YAnlkB7?V-vQ6)|N2R#QaoONjo^H!=y-=<-`K*~8Oi&Pu9k9(jS zQQ8>fu79Pd6gH!}n8C}~_Ao?8XhCPH~`X%E>s?Zm~V zR5qRTR}u{+JgMpYEuRXaRK24D1^IT3SG$VYk_qzN+|aD`YqK=U3^PKn+6vL9JE$m* zu{x>_IdreSr14vNM}Ab=@bLIW=6q3oSP-??eSiSDh6LQvXK99I6M`*j=bFu8%SK=$$vcJkCbQ!fn>dF9qF9QryXgf&Mz1W- zb7Lk8cTUN{z~N;Vd$hAQ?9#5?o4AXXC!A;DnVUVnXQjp;ePx~FwN(?$D3y;prv*|VWPl{R;Wqi2P{66A9 z?R)bXCrXBau!H6Tq?FdY8)C3T#+^>j+lGUe?b7uO$u`m&+(gRsBFTLe+K_|IJ0{fb zMm%WFtBLef@#8qBWcxO``T=*HPs@Y$R*fGlD}{7K8NA64VcR%Ag+$kB97;UVV5#Fw z=kpQGPW#OLb!^ltUir7)@FG>4XPpogMh-|?skHD)eYfEbMqr1G)7II4o_d1LQUZSm9lJ>q|wMf9#-kRPg8Ny-_mQjn6gjp#> zN2Sf`4qj(?>N4{@b%tR;j)XZpaMvfLS5g^*t7dGMOr>S1J<~R5EFfqV`}RXEa-gd%whHLRU0zfK&?(6sjRMuch_skRE}^^2`#iaTx7B96i)pgp#;NZe61*(pjb;G%)N0ZQnN2$2x zaLC(fHC*WdkTg?>=HMhW;>>d~=;tV?LjO~*amLC5yNTl7I8_FlDf1kS z=Sy`-!t``1r>!XKMkM%nRev$MC%eQ^U|dJvv+`V@_E0uAzfvxw#m{-*16>T`}8#e&bIG9 z?&0BlIv5k;v*d#j*r&!ch<)%)B*Hs&<)1Y`)rM%M+FS`SiBa`p<0axCYBSXDhN z{UBGB?9Lc8SgnT^%%l#0XxbU(wqC|j`%u7UcyYbg`XKWGXJ@0(FW%hgXe{yMZh!7D zNWVmSAY#{Cq)hgc!E5MZ&zCf8qNXM3fyJKOJMS}xd%c?BroWW#`({I-Pwv&jXH^w< zs%|Zw)>!1%9Ycp0!TI7(*S~9gP4%oM^=`Z4$8#My)i$9mcN zB)tKsH}AFjE;$7$wOH*cXF@VuO|F(K6&m+x%TEvC5w#qg^d6c|BiZFqU~t%F5lxxN zKQ(ALk>VV^w?ct$PZ`G`^@~g^&ij#A>ykLeom|bPxowwtfsWyZ&nB^tdG!@AH!kp+ z=0Rm&E7UCR*htgKS83Ter>(JWNE&g8tV2Ol%5mdyoyctuosO1@BO4FS_Qo&z({}gm zC8sxDeIS^@7&m`qF_xK$SLe9s(iK{t30U_hqsDa4t^~H7E>&2IaB+vmlf3jSx-W56 z7VeAHdQY#;woXgUq(RC=QrT@|4ipXSm9$^{SqLwX#K_nIIQ*D$pWKfifdDW&q&s=S zE=gT`Av;-r5QkhZ(2Ur{B@prE|)wU-(Jmd_f}(_*&XZ`f*TIZVEu@vg zcG0Sk#Zt!TNASj{Y`z7-rzbRuA~VG9J<^+ma3u@3xG$SP4(p7>ZO~Ld?e~#(OYgs> z<`?aN?e^{$Pq5I+_|c*E_`rSf#aq81V*0gkrn&LZHURVy61L3Fe0$ z2bmo0qK5_pKUc&C#mB^rxn`&5zJ0W%2RmNnJ+BL%ZJ1`6;SHG5yFJ_=(W0<)NTqOj zZemrgWC^ZR0!Q4^3#Z~Es~kL>5XQfeE8r5{OEEJMCj{>;ciw4xqpt)Vv@e~M$OiS6WG^P^>*0iXeX!FML%)^FgUV@ zBZICdU$U<<`J=G4AWf=lo^E2`i2M`i@d}cqw`?I*Jh)y@sZ<&z&!G2O=qcZvj*#U8 zqWl}=k!%MvxGPO^6bcB28Iv*f3{K?Sa&91=kF=PLexP`u|BzG<^RqOXPR*8=detX> zHT%kZxhq%Asu`V}7;;kQLR;hgf)bMk{nZ)1`v{U5uJ{lFy@L>9Vhk#Ctr49j)hXqmBMf3?p@M#Q^eF2lYQ7q=x9E~fsU z6ylFMD|imNUN8fPP2syXgxkcAq=29Dsl^DF#XRB3Z41 zxOLKd`vXv#=m1j#t1MOFJ-HzpxayMT3?cXAQ8~2?%%LHlT6XTrrxG*+<4q-j-XqdI zl%hd3VGdfI)4u$>6G(B>*`(74V2?8dsl!`fTvKu9$EY1R*a2uHmY8riA?^XwXerm+ z!$+hzrO{Tuc%|9fWkXPd)dxfEAv#l1{fz8s?--dYP7Pj&-`Crp9_(BhK;L6@d%4G+ zf#_Q}M@KfUWh5bceeEb-hX+u&L}4e2xjkLwY>la{PkQgqo1ImVS!C9)C}_=qTu5c{ z3-}OX`*;*GDyI1sTIgA@n3&(XF;xY9;rY3S+^9op;PUWp$+i@Dnp;i0#E8 zL+(&CgWr?aW~fWHOtx;TdxWY_a!;~H7e&IJ`JkOi=Z#kt!%ITy_W*G5l39}y)GLMA zdFsa;i63kttk75K4`7o@AQRmI33Ve4Z1v79VzDRYU!09IGN1Nq_B7#<9evY67jw4A z?v=)@zat+~)>NT#*MQo-*8vd=+(K#QMj5#+7z-xbcR4LHy2(R#_44cwzh}GBNi=$3 z%0hsO8&W&-#s@FsBqYe>P94vo;mVU!$tZ|#G;PyPT`eK;(VSTes?g~#Ih+pb*WOdy ze2vf0?#QYKdHlYqY-&i}d8Vr3juQx;CrJGz55K6HxFOgQ5+jg5h&M6QhXc*AN79~f~iDN>_UR@&!PjiaH zWMnE-zvM9T1SLz|!5fT5o2@ibBip0B`2fOTEwGZPs2|ygUOU`SVo1Vv*Bfq^AqbQi zZ_whI41BdCHsqxR!L@>yVX!&S6E@Twy+Hg9?4eNh+7~$}0plf{Rq2MxE<*EgwF22c zZ+?^TX?u{uUZuDs5WGHGa|T*xiZ;gs@5e(kgAhZX-WtGwB5A4+vz>=@*sA}AN|~F!7lE-g#X@I7Z`GdB{4!$ zX78tbhlT{2~yH+ZMb9T$53@TDUP_2;g` zYa&w9{gQVW_DbAfv>9Ir?dLgF_`G4)^Mm(Gbbp?xN6kOJ)o_M+IqTWWN1gE4o#Ok? zNq9eclZo$9a)eeWAH9$}(;I&MzWur+&_ik3&n4vePXwe%+E!>#pP$rHwbXhJRh!YO zUZiz@bMJjd{v+rGJI)q`G(}X9WP6&HanKAY+Zu0#f(@;OQUhgA$LHSQV2yCW%S++w zA1UhE8R91T7f*|$dddop7g{gb1x)xYowtUTV`^waw0BC`%=saED*}JGr~2XJ`wn5| zE99?wX)?GRcSz?I8XZY=IGiBs%dcg5&`EelaLf#|qqtSEeO+KhTnP2d)3YVL42_Jb z6dTTVN>>DQBdiW>?29YF7>SNmrEu%iPf;8@eN-naU{PS@@Pu9Z2jvFNHVZFBWP%;` zLud2K)FVmA%sTTuDmmt(J>CB1?>=6FFtim;tdZU6vcIW4tyw*;%&oIl|A3u$!B-8X z)uU-tNhVV%Zd$QeFU_B&+jGeLlodOMTd4Vr%druGHI;Vf)-26Ru6vBL=kqPn5M|Rs zydOFP*ArmV`^F#I4BljC)Q+fTfR_~=?DXS2s+4|OXI88JaNJ94$RG;+&`RFRo2|z?cD}lG@6a!^gSyPZ9)86&(nF#O z)wtu{QQoyB4!DiEBvWE`>66EHtov2HCw}{`g+9a&?Yss61fm z?Zb7WwBDVfwgH128=v#fQ{I`<7AtdN-eBk)>9P&Zpc`CLt?f0i47}Fy*wUb3h^I5n z$dGg@Q89Pl(;F)V7DYeGiZDoJ#^0gf=g6yOaIc$qa}7%!f0`nE1Lx7V<+>|zzgofl&4snACWTCEiAzw8MY}^Y36vR>;wA3IO~y$bYCR$Gub_qOGEtK zrQQ>6O?SfsECTy@!Wxc2bw5^nr;j-txL5Pgi9cw8%ZMqspCqCc=S(5lnr2Pa*$ zC6%1b%MAE5mYe9L(x#?}!_+#|&I3L}fAKC;s5^9qC zQW&)qSF6v6dId@O(%MfJVQx0YdW+}0<*D6!;L^T{l&TXIXA|0za<#eQ{jJ#)u((C3 zdlrwodw(8k_++soaA581yiimkDSr`8Qfi9UNRy5WI-njtzoa~#+NIv6Sg0{=z&U*| zjHx0_rWbb)S~;3|Mnav|+M6lBRbb*w_{tcy_KL@{IomLuKn+jd6354#ERB_PdTxGe z3)%rDVj0f7kk_8e%Ut;#NewR?#?_`R2Q4qpB~SbJ>Q!2gUiq{S&KOWivrg668n>c* zy*FsZon$0)`4FpL=o0;pX#z#iPVaKt>~|A6E4_()>Q^e+Y06vMmeA@S2N7ef-bsqA zjNRle9jhfU)~39PKr#pKM~b&&1nh3}GOHZS9A0C%vq8#d`}mOd+MbZbc^ANp6$vw= zV&w7VQG1(ni8LX!Ajwc&p)~P`l20MC9+;V8*{995?h1q3^ zcg?tuqSfHTr1Z}f^R=nXoF85 zX~^CdhNLV~DpG!`$m}V!46Ka*(}GaTv6F>vMd4+Kjq<9=N4xUVDEsARKI!u@cV3$=!-Qb7uKgEN7QO|oR1<0JQc5k>W?C`z(g_n5|oBAwMkou5@_>^_n{-IE)4C!rsn485q4>C}9M zvX08c%f(TDjh#7|LZ-(ZP?cBg)}Ev(O7e2E&DfowFUv4*b>oI)7LPPc?BjrBDoPa= zFda{bi4m-D*|U(W>NEAUiL$qQY`*By%*_5l+zAt%C15du-+hD8e9{Oy&t=(Ot3KYl*O-Q`ERQO?mZ(dU|Qo940eOh`BF)x~DA!c}$yK8t}H zd5t@{0yH6MiZUlcpKk~{GKQ?DT+|HLHO*9eG9k(E)aeI}#?d`aPH80q<3W*&yUjV9 zWQ*g^YD&>~-^%%ky55%OjzO48uDFfpCvz#t&`;!jD-eWh^~&iVO;Ss#I!?}{#U7_sGTW8m^TC$@*9!XzVn4tfP z{vKr~Q|(G?NB)M=qRr^1a6i|$MLL%WkD=gjU)LF4swKvum6`5hhX(n5Bw4-XOzxMo zgLV&fEIxu3hDEA6nmDams&aaI%+lZT`JTx~7+nxdLy*ury;&n2o6H zKw)Y0oun?v_`@DasRXM-pVguaL#Jeu4&J<)ZHVy9xEu!4cw`_XS1DfA^htv>Bgdxn z9k=<5>FkOgzmXvO{$W&Z;avx2GM8K;T7Z#SJyjUKepKODq>pk)ib@k(B3KX zkLoS1B!dAYUaC=yF_TH&eEyF$mI@CxMt@Qz)#xQPY?kii45e@J^aRFe|s@Ao#`WCrlQ(N5|ak@S5kKV^U>L@>qJnxX0L)y5Ti zmerK`YD93M^~21_**jFjku!tM)6lmOi{gvPy(H@^RX(i9N@$iqCGFhRnZCA|y%+Nv zE=sOQ8))#fXsOR@;UGUr-Y$Lo`QnrFj7xH~roL)?h350{A=um<_^0H8A1r5*T2Qh| zh}Fo*dx~u>PWK-2x}SYvs3z7<-G6GT<^-E1PGM2&F1w72OCYUs&20ASj=3c*H{1J0 zVILi&95eNoR$M+Go3jdSVmA-!y>TROih}Az5{h0N81SW>+cy%Hi?pZY`d0Y&=s$;P zWx58pLF|9{F!}l}zxLFQrXy^CJ`76WQ-2D1A(ua|?91KXUo7%QV8lga0jIEJcb|VX z7&mrLnnI{mqfOp$UAu(AhO5nLCh1_92>J3X=_u>s&pN7shgy6B40Fn}%`g?^ z6~SjHIv^J6ACJeqh-4a^#Hl>fy7+u&#zE_~+ysHWBU&244R^@1P};%?ysDQd{p%hk z%1ctx?No!TdsOx99-G+}7k{GAP=#gAN88>YT#nDB%_*Y4-cK^IjK7M-wWn%Y_02ux zc}_L)s)^5Q$#I;r?S3rBO59#wyu|6R;}A!6wpm7C?mbC5c^OnIVefjQPx;F@l4+Xx z&z#FL9D@@$PmCf11C#4e?PLBA&qKr`A7+io1G=$mZzgHHA{S3EluvDu`pr+c3Nyns zvqMTG?hD~2E!-I#sZx?7TxW}|nAT8anaYjN!d0@er@Eu&HIn-mX9wxb4f0%6JG1B< zA`1Es_4rN1ltQ803T^`9ZGNQ3XU`m?n@eAOI1Y0>?7UR^JaG8gW}?9?>R`- zd@_dEb6#@}%?g~Nbu)yg@5@^^F|2SsGkHyTjHNObUjKfwK-@X%%y2KYt&Vchvz{nA z9v@YKthEI1-Z+Os%adf@=dp1saFMtPL#cgL4>Txx-kz;GDGRXig02m}n9mD~#yla^ z5B5{CZBUi@oqg-~pmV2kDrL!otgNS5#*W-RLtbmGBz1a9BHPrley!$xwi`94T)m`J zL+x0&&BzC65qU2PZBlH`F4mF8s=}>fL*8~ENLQ4uFh`}WfZ65p=Q-q9+Vq>+Pu0C` zNtJ^xOf!1e(W>60RPyM&q%NMWW=!LdD}&K~gUo%n$kHCp52laH&sRR#n;q;W)1fTG z;IET+@JhNemdlxJ>5$I|IaJM@!c)a4ZSZqSk>sJst8LC2XF}?HZuc%M30z`ac_0U_ z^E{y3~;Plb`ZZ1Vjy} zp5NuMvCb&7)OVY?Ub!;e9WetRhBn?ad=;q2zsOYnX63fv*a7jOwPzR2sJo*UPd4Tj z2T*u&nKteWY8*c^)SFm)N{xOt&{prPmGr|m3*tL0sy**8s!a>nK%kHKVIO%pAl(cx z3?p_Vsn13Dmyd^!Lw!|pGETz^LrvWiG&{XBvZNjA)2qVXGUUrLe3al@ZYJkhJPDma z2h|T~&sd4}Yx^kHdx`GxJg^||lEz5d!oolPgMj<#E7|EOP82)kb}k0VS!^uXNLgHW z(UKHN6z+65Q~Z&14gI3VE)dNv%5VCd@xHj3c4Ni43Y^=No!0RtT;z)zV-SniW`fQz z`rG_BTc}#-j(aZIouq$-K5)<1M+1F**{m_!^YJ?flpH&*?H|rBv=QtwNdLw=O;L^$ ztJQ!Mqv7j}b~h1eNz#62`SQjQ(MPng);XDIM6xG;4ucJMM;T|~VyEOq#ip_)$n(POEaCG?kShMFr||){~%QcnJ#$u)zbXvhW-XWm2&0Jp3r5P z5|dP!w;mEwr&UAmtGYlIqYtntsh$amr0OaKWZbBM9#08CzM0aio<+6`rd!D(o(15tKhf3zP zz%%pJCMuM^yp*ocV-Ad+@`LG8@W;rHtzBIyhp`J)Mni$IMf=k!S`CJ#IOR{V@>UNe zdiRE#bUids?`6(UOPTJLV)r_osds|4tA<^}$+PR=J(js;tw4)FeGOh6ud-bu8a2fo zmb#)Mylk>)vyoLdHO!C^-@ShStg3*zQpFI(SJl0BlFEvVR8!Y-;nU9sTp5kZDAY0y z8->U}RtJ!W(e6V|2S_M`N6OybpJ|o~D_I($YGc${ zxFAoTCYiq5vGilI!Q>0m?;iQ6cQBUnl{!rg26&S$DP~Wx4lz6Ab~jqNXYEbur2s&U zt|$#Ra>fS^9L`u2VfQ+2?3poP%^oCi;kCHIPALNv(&INs?D zK1*v3fi}nuA$hQA_Ee57bzxx;0e%wAnlReWCZY-$1Ts&4XVGxo~f?@gImXs3{uP7s!h6Zg~1uY&-4 z57y}GAK&GG-j{U7nT$VGxOQNk*R|0-V%?Z+1*ih!CdhN$mc)ep!ReOuq&B@}?#7o^#dWh(Ie_@X-9R(2aF~ zoW%M*{PakpI;!f4mae8EeTOo-y^YkRq9{brMw9K8M7@@SsHEe@n0Mywo4j!el1V<) zA9s!MatWufbM1V%F0z*0_h^p=DL;Nxrw`x*1w`D{A^i4^nn$ z+vJY$ktu6`B9*z58!Y;YPiLH8b1upao-xNWh$DwARvX;SC7~?*AkgcRtuAYdinv=O zLg)TmsW4pcnB6Q(eQTGTZ@l1vfVI;-ik2bkbyr+RtaOxpK5)F#dp7_m!4d7IUhWf~ zn*H7~0#?3R=obe+3Xgrl1w9%sh??nXv1nLKOG;UxFB;`^i8QU<-*(9U@tHaL+LO~P zE-M_14)3^=_$NsCSF*U3inCfUKTf;8zH@y^fw2Y%0%;@3j9iItC5`gC-b4Fz9%kC` zPU-EDrUdf6`wCr2+*iD6y7|Mn=K{M#MyfvfJT9qbqI&eYncQ#>`_3o}xQaULE{9_v z11D-uMO2ce@{?D{?0Qlnqx9RAF_Sl>8Y()mB(@S-Z337JDo|x{^TZZZ3u$m7ZQFYi z<+nLCXX3p>LJuxJ-BC=l)ZJ<^spX}SqT&zu`f-hhRd0QTD%0NR0`68VSdc?4v}_u8 zP96X!jTSp%`m4`tPFfo;tf2=5LZ*G&bI|m|Zx?#`p-KV#7mtZ2Ybx?FB@76-Py$v- zx$EILmDX4y*tPI%- zwZrAT#UuTKO&V7q#-~%DuGCluwGACchLb@jJQUw7?^v>C9ysnkmC$E+?{p2?GUASp z^pv-lB4l9nwtZP91!q8(3b}6(KY;DEYta2eF%09KB$d(zjWJ3~59S8n@L!nceU9;) zA5o8aVdinymMU%~oI{nDx(4_8c<1%vZXVC6p_C{&{?%7E1xAD(`#h_RfCp3^1~B9j zjP7)+W=fi3^?hiLWwpM@{qtD)b(@c)O6|7rDeer7z$-;`lf_D~toc!A3P=Z}BsS>5 zxWEv_gzimGdykTkO1fp;fcIto6u#Ixx&a5q8MK?L^!)(J2!oAVQJQobU4EPz0ZJAP zJ98B7cViNet8?9KjXRzWc+b<&zZ7cHTA6OI(jQYn4HZ+EAhQoex?N(3KmSamtR<>r zY{fBcl=^*kKcr)Ejhx^`Y8Ith`w*k&-1Htf!y=1~7wBaRwrjFLJgwTjNPipq;$-7I zZ&ivxbIJ2uLw-t4@1C-Su#)`TlWh{keTv3>RZ6##gGq+hA{co~p&w@1eyTaMF!|m% zXN*0=$LTtYXN%%p<1)>rzOtMtv-}3XzMq><%4yHim9=I|tUM5RZb^4af6(K?K|6dk z*=|`&D9u37!u371^fJ`4c0)FSIdIr;_&5oa>r|l%bg6=T)hWG)UbU;|8qI@GX{If_ zV?y5E%9tgbp32>TIh?X*F?L*~D}PkA&ABD=`N5Ab0wze#vH)|8PQM_0U}uSt=a^I5 z1o@rN_j~6*>q1T_YJcXOX1pv|<*{=lt4QY6hOb?a6%Y}kfeR?%oahOus84Ib5JO(Vk}(OrjQiHI7lJa$*N&v0HlY z%oVS3sR3Q>YN@aj>R?P)JGd-$tvPBzbV(2$eI@~NE#gFKP@}@L4C==Ds_Z=Y9x0Px zgDVBhZ;zKV%@)iG1q;R1SVyh6KX0gkTG-81FAgeF71q^gi*$}cu2Hy`h5OQ}sXg}{ zHEfw1Y^acN9MVdC{WNxt^u&{aK{!&%SuNO7VVqUu3!>^G?!Z z^`9>B-U(2l&0G~5Eu`p4v(ZwCJV~Qp#^bvZN;Y~+X)dT*zR^OUVla^A&VjJSlT7pe z3huSr`nQU>#0hJwRQe3SmFsMt>OZyX zE;)_=YnGv((o!CocSROdT;&~^_rZ0U1HPZP@np)A6E<(tE(v-}Ia4>I=o%uWANGh= zu}t)6t!q7gwP@=aXYI`Si9#lS^zF}w*eczT)!hm6?n_BiwP$t*S-IMo%7$*(I6oYA zh`M9m6@pUhoA0Mus&Hlbtl8B|^Ry&f1BD1~yL>$9Zr#cwYs$*Rhp+<5xtqGJN2}${ z@?J8nN(@)2Cx4D<_lJbw` zJ9K(Ul|9!Nbosi6ePRPX;Z4Tmcwi}8@J!G>pUX_Ry*azpLT>f_1n)aB+t$dYKVK*w z)TB7q4@-g|?+%g@Ia5AQ)UPjZ#b93?`#my+}d19x?+0IBinYIL1{?K@qc|*hJrFL^kWYqCqL`NfrD=p20V^3F2er71!;h<~oP^rmy*Rzj| zR$tZBwHBQ-G7qdeF}81}I2X#GtlPd{Qo8r>Am zxL9~FM#ri+NHhf%y;dAp{F&CKqeKlh#E45RtDNm&G9Ni`SmX&1Vp(`r0X>6h7~EBx z*Qes1cA-oCN*fsTA;=2EPUwx*y4)X#4>@i`LAKa$n-DozpQC5THX*9E%ZtA5{Elaj zt3`wpR12j~-dJXWNW48XuwtMKlPt_CK9M}QC(thWM_t%$RIf%*Sja}br{Hn_hlgD? z?bxp#Z)dNLN{?OHrTMDC{Y`wa;K_^#i}w3ohj#tsVZ(aj{C)G)igG44HhC3XZl7Un zZ%AB4zHPcH8`F3W;Fho;0TFrZxr|4R;|~`IF|cdJM*>L|vOv ze%b#URYV)Wgq4_fFoe<)$~v;5Nm{01pp0eC!@2PuY-2z!|L`{kBg&fMu)asvJ?EGzgkJU|wL>3E zuAZNIIGSl;_3p)ZY2=h|K9lZ`sgh!qapfIbaVy(CAEihZeJfj)X-91KOEW)euZs<_ z?~dL)aFOd+4B}~;@xrxkA*;jg(gGb#&JRN*hZ$@P@v$Obj1@ER@##w{OcE=sE)YRl z-2H1NFAMGlnfHy8z5hJ6;JSu z%+@#Xo!M7(m*>MVWOs?XfV0iv`sYr5AIsB^h}<@dUya~ZJ^gW|yw>^{$-9Xm{SBGq z%cY}DKJ|xe_>Oj`Le_3Ym-?mqwJ5#05vHQ3I^WhR*y!5mhaPEQ{;c)#oy&xGr~4|f zRyz_ErdeH=u6nP7^uxQ@HtVo4*Fzsy-cZmT42^Nwxa6GaC*l*J52v4PS05)CKN^El zHGTq}+zYXh96M50pPaN#GKg_`WTG=}lc3Uepy$~RZ_PMVCC-epN&mI^r;ywU6cD|< zug15@JEAQvcA3%g{_RN&Q@wu!wpIPI#*H`if8p}iy}r#kXdZs2ic*g=IGvpz&qGc)7OvA6ivP^Z!k2G%^DtJnGwBHx9e8s zo?|~jU^DVbef>jD#%Du0LdJ;plNo z(Z*S-J_!cC;H1g!*>eiEyz|3x(N`zl`i2d|`7{pzl^%0ep7YSQ@pQJ?(U7Ixo8|em z35dTjxTor06T!flvBa_REL+-F>XyfRa!^Pavi%y;e%|`AbCoRHyv?&q+J?b;Wwec- zLf&fU%hLG|V0UR(YU=j5G}Rm{yhtWfUSBKNI8N~tR~p}b{lI-j=wMJnbjT^1{W>Gq zl)T;f)~vo7QE0k1lIVrW*AYgQ@gEVYZR?6{8@}8NReNnpA-rOg>1!W9ABJ*$@OiA( z8CiI&I-TBOWtJB*QG2;@A-gOSmVnw&pz3jHeML+@TV$ajrq4Tr01eqa&AS zm(oJ~1)yg#UVHZQ9U}GLIN7+Ly_l&~3Rp8eAJo!UD+l3S-{~-i{a7yy4K&)R!r7&C zPfdrZ%_B;M=_m~=DTM5X%#S|>&(#{ge&k#<*;}b8R-7I%bpORHter!1)k@n(25R%! zHd(Qm8x_C$F8gk_r8S$5vIVRWb=)&qlW#Sw5k=#7Qrk7fDOGVw`xT2EyB}2OT3nF; zifYNI@eBpr!JKy}I3n@_ed7wJ*N%Ifj>}QEVL>NcLyNcv5<^}Yp7VQEwdV(kp@S5~ zy(?`!*#F1YS3pI%eQ!&5cMheb(hWn0(x8IU($d{ScMT2FprV4bG(*bJ3?8W2vIx6Ju zx8vHhf)WW*s4lCeck}`8Z}ER9O}&9>!IU*$v&wbHQWZ1jpm&9E=2#Iv#XF+YTY4Pi zb1Y+L2E2Mbu1m>;JnND_gbI5a>x|x=0=V4_zbGjUcUf-CEcx2wOX%0-0O+uK72mv1 z#HtvfSRv>L9f*lYd<`RKh`i3LU03?jKC$rPp|fKwfQvp`!aC2ZYwb zD|0B`@1YR%XeaIp+&fB+|zNdc=hBy~@ zxl8(<(P|Wt`%|GCZT0w-;=ZzaURoJgOyIR&Kp5I=)W}fI66v283qIx#Tuo~ht0%M= z%rE&6sPtsqpR!8`?`5RaQ_#S${UTg1QmT{&#ZsQ@fzuM$Gs+SA&k)0LJ&ubP+w2!}Sa9QOOrxyWO#~kW#AsL(Y z6oOpw$&n+N20V%B>!lQOz9C|;U!uKzX-vi6+svk;t7pd?rU9zDkxBMw_2Mrc5MJ|B zGQwHgrI4HH!P`g;2?Zu>(_R$zlA&%il!Cga=w9FkCHn@8?=qOii&ecBThG%{rL(1JAWxBLY;=Nq_#1{RCo|jdHw!SAt_&$HTa_qxwX$$i(?D0a*s6kf?W{#W0 z*|%w~n?T}*I+L4GciZ9>Y~P0aKVv)H99w^|b%%yAoQ3I+adk>U%osody9b|JjSFLB z>$&;7b7l-Vj})Y&IX$C!FZyzbI}hha;hea1Gr^H?EL?cVZsoG<>0+C&&Z1+6<9Z6s zkR3Ehhumw7QO!G>WT6HBwuz(*M6cirS!Y~n!CH14m+j%iU9RqZ`GkRC2rXHdai1Sy zNP)O=LSt(fwn+C=X^W@4iE(ufx*MOA`r##WBTpZ9{n|uznSCPGCPW^|-H>2a&jy=8 z6TUxPVQ!_n%jCDRWE{dsBo|OUAelh<~znBYs4GPPS{ zamWL&+a=jl18br4J!4XjkfxHbPM0?#8J~%h@Hw4YQ&D!51_DO=86n7rE%|V3D{6Xh zJ%0h(s2w0EwiCsWT;7^zPs(Yr&918fyDv4vB2p>=02k9+^+RzMqX}n{h&+QRwIE|Q zd>QuGB>WkEW)vROMpitr*c5^;xFsapjOlf+V2yupdD?Ft?c{z-xI80me<%l%l;y_< zHW%CxzK@zK#@t64QI;cgm0qLlCR7hGt-v|dW{|=_dPytJxG$#3t%*jd^AUd;f&%v^ zQ;b=t?LqwD$B4k>l=G;UNVaQfBLMBs~3y~>cD?Fh&yn=iHZ!s5~giKF`0 zcMP&;nd-s#Y;YM>jRJro{t7YYnr>h$wC!GKgV{2$epFVjno_19LjAWdV0rcmgwip! z=-WH9piX&ixCn^75tnu=WQY`nDb?|2$nJOQ3NPx=9DOndR6p3H$5fJbkPP+Qnf|uT z9y%0KvZ|7SR8$MxdfBZf@*_a&{P9}IY`1Lg4jajMZxAqDy63VyyqQs%WLU_pf>AAZ z09KT_n2BVTBL1lUOBT(hK|Tho_2r^3zyILzcaW zvIFW({CQ!SGW);;CHxvs5+)p?*Z!+svT+j|^jn&B@k}d{mco12&E<`&t8!8*UZ=|> zUHbVN&GS+nk}2m#XA}EA;R7a0yyL6t6hq?1;x-AQ1+JT1Wpy9fa0Fj&370RpXN(cR z0_iXcc_U~*uqJ^G*HYit71vl-JgDO+#G5tM>hdgR_%}eOp}S66GUSDqC--3{=iD{d z_-`Mh1NN=lQeMixQ?f~!Xv=Qd_>o|8w3xrT55TjzgSZq=>uwB7V;Y1D{Q<4k{ws^}id5qFf^@M)FF>7wt4T6V9i4VA8Itfbjx+fzmL1WP zM(Jx2_xZY=?iIahEar&>tV@!GKK2L-2?QMz9;qL`a31 zo#a1aRit2DmaC8+Ow@-5xM;VC2i;zcm>36kwXkaBu5{8B@?lCnUqeD7P)%5b*3$9a zqlZl|$G%zM4dO*ad7%+QonDyDG~Nx22?RePu>pkpA$Guetow>BC3vvns6!XD2pD(G zmXagw$KN5~C0(3&L<%x*>cni0^9FlcwFjfw!~*ewK$bC)*RoT}V~k9n*0;qkO7dd~ zEQ_00-BK`V$vx$sKwBQHLNIVozLy(YCo7RouXHAPpCa>ZlA1zF4H}4(83kA_Du#i; zC=8D>ScXgP0ZbYLrN|B2&XffV69!6;HaO8+>*wAOA~mRMpr94j!Y}OOK$1X4mKg*H~Xv9x9An*-uax zQ1Kg0glQ91xiXxWM2XchQ2acW7!on;>;3LSug* zvyqXU>0AS_#T{V9qZI)Y42knRqTc@yv@7VwfzaBy35>1i7a|VT&q5IJMRhrMNfK}X zU{sQqoVw7F@%HhtjB^|s7X;4JeahS|4fn;NryrD%ezM-kz~kKx-Hgk(!-d8HT_>Lb z7oQcsRPWh90YMtgVlB9v^xJfK-gRUYG3ve7ERYST=MC%Y)cdTHb&&sR9ie->ylK}~ z3_gj~;{F^RIpK2~t}{b*XAj}5cjpC)O;kzO4R8pJGLOvDEfy!Lu#&iUMK=);wCz8T z6FazgtDq7avw%W{qSQ`_z;x#(lJRZSymxk?GjGAZqV*1Ga^LPYBstFajDvAgQ@S>K z?t{P`TJxDA%<542b$Ur3E+s#bwMd^>>m3wV9^_Mwz^lTP$)PtU^2~r<7{+zimC=)V zn#Scu>TR`v5MNHuX9q?bgB`n91DW^W9yHv>A~i8~?^e0A>mVV>d3+Jc@@%gurIzqe zWpIT3A?ODG>Xg@Q7o$#snUGcFvlk#ch&K*{+mlv}d5CR!$y1V_nX(|ypQqKpKL;f@ zkB3!-NL>yPJcQCWh}2T}B<;iCddYDv zxy=P={(Vb{$NR#c3d`HA@Sw4vlNEs#ylwDZ$2}qlx)5iEY0bOF^BkoPL`kzBm#k&!0c6UKzIj5qZ@K6s z+3ibnbt-l=X5I~u6>|x>3$L6_q8X|k2nRcw7vK#Um-^D>I(|Xge(to@93$pEQjDyu zqhVQ?*DBXqkLg5dApr_*Uds^PQ)Y_}Bg~se8+&@NkPy8fN1kW9@rmPdTt~w&PUy|; zT7x0QrxK+?B26TAq+dsuLSK#ZvQQz96c@5$12{KC7N?DP-iTh{widbHV2&*IMnpK3 zF=clokm?E!l1tqX(lc>_+Ad6TkqEQ}VlUx#b;wqh4ao;)!x2k5pX5-YId7AR0&*oV z{iV{Z)5Arp4Y8WLm5~5($B!{>{3?vmOfgS75=3=x(4^)u#ND?@!ytnl1Q#zc;n-oD z!t_3BkrF@T74qNVgo{25!S~@Bc8@rr<2CRG7kRF zbFmxg#!)ymJiZh5))R~u7X_j#z3n;<{uqpIRtoVr&~zbB!SLe72O}`Ap92Xz9~~5| z^6$%Cunh15Xdv{r?C}7)JDqp5I;SngLvC>G~Ntw4vJEL2q^p2+?{W!SM?IuSy@ALV< zIWwi9xMsov&CdwPeQ0IlU`RP$ifdi6$-KH4S{VP!*iGSaM{JHz&|s+R$FKUia(gcp z=XRBkPdWD3mJKcqhw*duv|%pQbQSUt6)C_~e}XN<;c}webP<7;#zPU#TDDN(PeaTW|~w{IEIj16>tPo<;B`OAeZP z&a;rKvW!MU3f|H3gKB(Zq6f&IrG0)mSQQp4PLevSbS@~n@(NA5T#gbKekeo&m>eos zQO0jCh%6X8r2o9i@_*Z(_ z=V&r^S0md`+9F()FH7rTn(${PCaET7i_!X~14{rs(W{Ha_K;OAUh!BqYBur1d??x2 zISf7L$+#%#b^IyYx`)Zu)&553EuGn&O1p~JgBo!{zkzs6I@;NmHm)s-PR$cZb1AIl z5`H&iv{kfPfr@&P0@{=C=Gloi7ta23TZ8;IxhAnkFLFr0xAbLNWhwkwDd;zT8b1j6 zm0*X2qZb=yeImD3h#&#F}Dr>&|8_^ zB^Ylwju)!UmWFT_#ggg#N|C>jYqC~%g!PaTO^QY!h%SFVsiIMY!h80eKSh~E^#`h} zj_pn7l`NEaY@-st7X?7`Lu#SR0`=WDcVkgG8lBM*URxTx8(K=Sc|+oBRuQUa$<9b= z5rn`qQ=N@j&Oe=8<2~N06$ai>9Jq)|o zyC@iL-?ORQV?{s#i&4b+R}3aUs?msb4BjqGDRt4vVB~gi@V0($_Z3OXH90T0uYa^T zP&nk8CH@5}_U04?*>KgAE87wG8%2)Z2NV`6EBxa?tN^PjQ`%Kt0>9G=QS<3JZu3KC z1C>-~y6=tag$@r(qsOGzB!vJgBTsF&3gf2~x=`Ix<9+uwOh*%fZyznb{%A@!c{`<% zChUxN^#WW@!Gz$fgB1i@4+L%J;a|Bwh-zYq(?Rk6(M#AF%I}tb&al<8;7A$mwT~6X z|H=>5z-(^2LPUCw9a%{hW#eSuS}PQ@;X+Q#ZYecUYEA$q9!3DR%aHpXb$F82H4X( z6?dKqUzg|VH~`Ov`BgV2oA-kxTVatY7ujki^Z!sr0*Xk}iWM){Xb*DEhW63w61qL)G z$iM(O+^;%>Ya!0LH%!rEn}w7MB95Pz;8L^`(ubcx!ln0~EIxav3shPzNN!>R+j}sK zhuBfexE_RP(67x*v|8|t4)&q()2(b%TrF-3Pal1MTwVtR2yDF93-c>;J6=K?$v{Cb zDW}IZHKzSAB!brr_o6Hy37@fQDv|)x`?~-31q`8>U6cE-U@%{TM**jU5oZU1oe_l8 z3bR1&dC|Ry7Yn;o{FW=7H%JmGu&K_5vq5%q=lKh8*S^}%vrT0lP8T8OfddKp?gr{B z*0lhF19(xQK}&QP;Kia<*LVNVk4;rx%CHWi0av&RGb2 zFV?D#raDSWqcG*B9k{*0RPxRWuc|>yc$_)6D{@AC&a3|W`~cmGqTE8gY%38#bzVK6 zFTW_pVgQ=R4NlJD`2)QU&a@@-aRLNaltcY!Zllvu#}Jt?1isz0Wr~?~US3(K&ZP>h z;6rDajyZeNBYK14>M&!L#WLyfN_u%qOm?9}sXqLII$jz`XXM991qyz$W=ao_?xx&` z9SHQ8a(qt4KiPId0@LRqtmlK?Gs!4X_4}j3Bjf2E4%eZ`4fek5D^6$$W=Wvi0ZzS} zvhNbHW_jgrz?k$~MI*-4Gz;|=%^`PA6r-jmQB`k?9v>sUJRs($mgx-IR6lT5Z9XsP znbq-|jOoRMhi*GCtw;pcdPwU=nX(S3z=Rl*?&nj*F6b}hM4Q_D!L1xwb9p*GU{D!3d}$0}2wF3euaqFZzfASoL;T#P`5r@X|n$MnXE)T$94%YE;}p zZY|ayL6Gkq36msS-y4e=xqnC!`3yi>c}^GT6UWth6BNnx+VT*%jo;i=@qpWAa3&uH=p)AfzyI{I*Yw;f#GCKLCD^*b z$|+!u!uJJn22WuTNj_FfvRHe1Akb?TXa=QdDo7|7!XGZiDn zMt(oYwm$U9e&gJ?G!lH=|H=Oh^9GOKVi6~tKY&$Le37>bae_@J2uNtT#PY*{irw}znsQMsy2= z;9MfEi=IHn@jMbe$25rOgcLV1`(mka4Y4%70t*pfczr651Dr^h^o2*9#WGOWDI1A^ zp`9r6<%c|CBv{>-BTr1}$Bg8%^pk0rb5g$%dB|o_X=IBWGA!#$zc{Q&1}J){cP*fN zQ1HLINXl`)49ue=(D|6#i*gj)D&|$^x$|?R zA*sictanXm@$?kuh?;5e(3MM|O++Ys$l2*ytd5eLl{*sPcifAbcF&k_dbL!*80`#e zVxXs&@UxRQC;lWtzlg(+8h3<+Nffm6eerR8gv^kNM-QLtGdu9G(B&KwGg9*~94UgJ z_@P#JLVyDEipb4N#Wvb7?~N!JZCf3!Qz_cKqYQ1y$?nWK+p^^;2IJ`)){ds2bmviq z8@f&EscI|kaE!-i>@?;u*muIB`WJIOYgo6@g6JYx^`qua5!pVJ#t#P^$+|9Iqpt_6 zGrTdPyXA~@K5ma&#x-@1C?_|Ec3M8&qQ8=pd=hl!hi8t~VPckve9IlGPL8HwGj1Ds zmzX!B7H8*p*NV9>ElOuJK4{y1E^!pu{vZ^1X?;&QO20*4RvrtxL?-N-d9OPqwBM{H$8;DV2-x2Vjz`zg@i?Pobh{62IzvFl3yA*@>|ihdji zYh`F+XU!94$SJO>28NHLYzqZOKaAX~eL6;2HaIPo*n7(nD&Wq@+~-AiT}JWV-v^0) zD?ssC-6TCc4S})=!1U>(GdQh`?!8WTew&}GPb17EB~{jr-NwMbp6&vsa4BUIZj1n_ zp@lc6A2;!%@swpccrV_QU%Z=e?YgA**BY=T>e|#>x?msP_py|cu5?3p&u%7Qw9lr; z%tJYpbm()XDQPD$TRF|bQY7DxNm*^$FOtLx2XvEd=Y}`yos;xQOF`PdrBj>}#aD`4 z08JndO-6_^(osE{rH*oSD6h7V9#!rc_i*BWiMd=Vp>%6>ts{hHka>-5vV#F7fcCwJ z#npHzh(-X()hi1B9xSeIMs*BL#!_<)rR;Ou`RhjM2rtLd$mXPjMLHSEZ4jJ}!uh0W zfe&zVUZ}SB+Ds}y=3RM6MWoq5zGPYUn6-V>HXK7|QA8$P?(Wfpenb?d#=eX{v^o4J zB!Kz~oO&E$8I3qU5>POEz^HMM<@8{8G|>get3d_6g}^>(8pq-%LS;a$rb2+D;3c~? zJQx3};JIyvBJ#C$#R18Eyqh_}g=AR}|BVA7bQXc1GvfiMcHKswmx3}_;nv^$O)(*Q zVo?(FDi@79Xf?@XF22bWM{i82cN!T2I-VlfF4$3iITvER$F*2^k=N&elj1&}CpjY+ zMdF=_74q@D_~E}o<_>DhL6IdkvswpIwkdkRJIW>KmefAEan|FTKq}CN%aW@x=TTy1ie8jCJ$?Hwd~ zyal@*i!Xj-tvPX=8|C7h+9Q4_aMw<*-qQD)3h9KOXI=psCK5!2b>_*B0dG~t== zc+FETu;TCSkQlm!Yk@|?*$=y$lD5lmWG4lEqS?+}n-FaEeG?$!B0Gl>cuBw9C8Tx# zARF%{lKvFBe;0QB4xsNWV*DK#v>D5HX}VMlg=$BW2!Xmv>54r&n*<%@>;q zh&mnQ-bN!{drKGV0NEnaBeiOji@^7@Frl9o9zwN3U@)>Evi;n6*TTr+@# zJ@Snav8O##!5%jt&qN9w9qvaIdz?Xd)~|``Wf5qk z4l=Pru^k@53*{d@`bTe1Y)4XyjN_gRfNe}c;4QB}8@x_YPEh0kwiaQnd7eK4Jh0Cb zDH|l!hJjD3hmBj-adC{17IYsCtSGFa{rMbS$l?ACx6Dsza)4JfoH1Hl4x4GvkOeD= zY~dMC{v(qQ@iVt@{nNrjGqO!ag8*S~=d#!cC5M^SFNIURP9xO#Oq90`T)XI->OFR) z7xUP+p9e73z_IbN5;w!)NLKeDkwx-FSPBfHo2s%<>us3@gMH(+e1bBwyBK(r^TI%o zuUhE~5N9V9dh+_p(>{@_OC`TD3ac5Xaka4Hp!*+VnRFx00}4s^;SVj_V2Q#b#j^Dn zhZ89k^6be7_7)brU#H!vXJ-rZ&3*Jo zos_*XHzNQ1?+?Pt)qxN3f6>bRV(mdZVfaj>>HU{nAVSDui0p3y!@usb;~~){{6VCH zc;T-Wzwgw3_+PLb%VT5?wZG&A{{oq#s)c(f{CVS#_XMSc1;+mc1!Q>mOGt`9mKgcJ zVS!1+NU0xw1N}r0LV{6nZwOB8ph;pYI?Z{iYw*0i(+ml|6Kx6D=|XtAAVcbzi<8O5JJs=b@XBcqR7o6>|dPb?@3YJX#W`s zaLupbu{nHh^RIgXn4kXSzWhUQBqDMcL}*GdR5xUH-53L@7vbBhK9Y3lG>OyBO`xQ2Oq`#88E^TQI=* zDX%8ouS;1}EZ_CB>u0h2zb~gsx3=lT%F=C426(hHLzk*c4139=nv5zJzheZ{>{y4( zTVhn}w}41WI+$nF4+}sS+ z?LO{&@6w)sBdG9Oj4}tErsx-yFDU<9?7A4GbWfX-%qa@2JwLix$ws>|=s{&_rYxM= zP1|DXy*{LAm;Kn*ZXOK*_Mu<|YpZGDlA=r?O&}Ga_-Kq@7iyp-_tEe@PHJlpd zQXKod==KOCUkCc_3n;xRQeYJ$$w*IPQxGMbrNwZg?o!){JFnb$o`mV${#Udckk(*`KHL_le9yo+uU4=NtdMaj{`|Mdl%)MJ4BpSIsf?a zI@u7xe*81-e^sH(4b?hf^vMVRXI$3hqYf4lqB>Qc39p zo#Ws7;TKvpk_B}`KEI_8sGxGqyZrPQGKanLagTVVz#03BrGTD(0dMeE(D%WXs|xn4#ICPx-EK~ zVF^68%iffJL*>mBF6*{(lF2p&*{;Zz0?-Ak#3XSdTNMrU*5*VoHC}HE?~$l4_~)y4Sd;nEEGoVl~KqlmGu(xcXzI51;XRaP7XHu1jwaKYR~p;| z&Hy5+8Rz_GC(px_c2(F%yV)si^r2u}Wf95Ag-&I-k zs~T*;c}aQaA}nN~og1*LlKjM1jzrY|Dz+ECdG-3M+T9QPriZk2h4B>ug4^HB+yT6U zereTe-E)x^kY`b`xU@ixyX@LWiQ2Oz374)7e%Ej*KI<-t%P;HZQxz7i;zesl0;rHp zNrfK{v%VuM!}`q%;jyosMpjs#SGdmJ+y7+Vv68iT_pbW<_PMf&5Kl-L{hmkeUPVlWBs+3UEm_Ai${UiIs$JRl5 zklf3%d4}sVRy%hXev#i)$p`x^#nE~hy3B)hE}QEUjHAwk)~j#&_h(sn1lQ1Dna*yN zZI6AvEgF>7irJ@=DNh`{c0SL4orukDtea(!!lU9mrIvb(Ltf7J-PzP8!wI{X`&G61 z(USGGPVq~!as2%Y$2s!LvxoxV8e3_{bm#=Z$B<3Jepw>-PfK&()Bsshn#0iPojY+= zBRD$f9NXAimPsigC&`TU#auk*#o`#GjwKduIWPlWsUqRzahg5dPD!=`l;r4Dnocg%>;~@41*kfF ze$BTdmj13%xt%uYp{viOZTEL)dZVpFYv13Rqb)V~!S1W+Z@lohJxUVAJJBf-!VQUh z&u}oYUHFprx#L0-XimJZV>2WaofXfw35U^u3eGwRE`q9K###N?961+u7NftyqwciZ zJn_Kr-VKZRa4qck6k0cp6W6^fewM-KyR~rZPxbQ}1#r552zh*j6L4o==y$mLV5)Ym z6#~$aASMuKd~xOGtD%Uu%atSL-8+8cdPGYyJvLJgwDhvO&yYIY5AYvC`KaGUSi34@ zr`bv#<9D)!V;`H%4)=+EoCUN^WKY0R=?v8o@&D|%D+r{uH#S1K`$T_sXbFANp{pVL z0#hSm_MzYDVa&$_{{XLH=z-bfeWrw1mS3i+`~`LU!J*1~*WunHi-gwI^#uxSy#B-h zuGAu^Mt`H4uf@&+XYBhN-+1L=8GbzDbkLaj($UvPd5YGPcVp(GH4`0veN+-IAQ+iy z*@b0ahWcAwPWGv4ixly|N7v3 z;6B6Nns0gGJconJto0lq;cyg^lU6p(m_k>Yec3*}&Fl0iCQMwFE#meX#mq8b%JD1@ z&OmZ4VmBaLtsm;zpTo>4Yv;Vsa{ejvy2hsCS~S1l8Q0IVuFoYaO+cNn+mDrlBBzPB z4DJ9PIlGx1z;*9k72+JNNDuVNK?Eb-cc@T}Z@ft*co#Lx5wg}+>-F{LndM-@ly~=8 zM@~}!g$HaR22`80DR&5XwBd@aG4Y3 zW1I7qlvn9f|ItG#xO~2rl|i9>-@gu?uc|}oG~KkHUuxL;eZ{h$ zI>g&RkJ>W3W3zwxgYBDBi_N2NpXlZf$Kkx?0?ikSR*5`cP{#^*zC6Oh!FxBTM`mx( zN{oA@UFE(mV`}tmJ4WTo&4xvzjLXIBL!qo$9C{`cp6KiFnEOL{pKa{$i*1jr92BTV zcH(!=+N{)xnYSst3qtuM9bc0%@afWnFRL!2kJbm@E?=KPg)$mA8B=ybc&0BQXP?Kk zw7RQea+CFwrzrH-^)^4`y>i3FPZo_4 zQaLN7dheHsd^A)4y~+YA^Evo%Az<>G%gIk0Dv>w2&Y(e~`lsYvGaSN3`l@?Dr6EpS zH&__h7)K=8SiWa;sba$ENw4*6jB0&~VLawkyrQl`&)&`Hzv4*cWbs?b#>WS72Ht)$ zY)KKQZ;{BR(BiuqUM(d~@#x?so?O}1_NQ_twdS0?MF&A%f4o#wIs$_D$fWJITu#=Fap zD0Ww~r^ICL^D7bJp0oZYr#WUOS=rZL>gOE-bH83V7Hbl`B3C|~SisUD1{Q}MSqZDR z49*asddU5IhA(02T?>g_-CPsRzZh#VUy@cH*h7i0GWmR!^i|XTvq3dka>wJF4`ol@ zE=!Dj=D9iVeG_G~raxWjT7F>Y@o~TTYh#TAh@yz^VEypX%I3C)F<%oN<$4M3Fyt!i zdBcZqvrheTl^%pAR0mBbTf!Ym$K++^-%!VTlZ!u?9;b(vi|ybXzC&p>m8RueFMoK8 zk-)p#a`0S*WSXSaTeI=#R{OIL&gO_!{Z!@1aZ{arC6-OT`1Nr8u{g5%mw<(8^kYBW zNfOATw&t1bd6p%Ur$*W${!0dx{z)GmI|WyOMBIFHzI05IfB?cTy4A9YYMvR0k=DE+ zrr444lC^&C$|{@sOkt$}!y&2a!x0W%q??V7O1vFa*uMOjNql^KT3VbV*yRIGPtLdg zclJX#rt&jSUX*Lga_F*g2^;7?FXJ@sp>D9^2`SRGysn~=UF zL09xj#T`=-k!Pg6kL`H~+?N7Pa=Kud)qA_CT)EK5#|8{d<{_P->KWQR%bPds`h2Z0 z*hfa+2l&){AJtQzy%s2CnoJpB{kqpWBJv=;Z2C$2T8uk#uNuYL<0-poM>ng74dnP_ zVjjrb4IgaWSW(y8ob{)=*vI{nhvA1kooXf2>%`an&4?z1OM0#kD`hnl&sUV$Ctr2W ziHsi})z|7N7T(R+nhMiu4={%nwJPAO3IJreQUgUTXwdTSaib9V>i*><0AjANzZ~y|O5X$uh=mh<0kgA%vvKs_!I9>YXmt(dytsob=MPbE#dM zt(md;xPab7x-q|#r6!H(LewxS{*S_D%eExmF6)>aA-AmfaWvyZ4kgAS{uKrRbAq6o zVRvgr!&V=|dH7jU<{H%E_ ztoqccvBW{NH|=4FQ&{#)&!uzc_~$pq-5%dABap^c`{@H{3WL07o2UoMNc#dzb&=v& zsAC1{*~dn^A74Lo{peN+`6>;M)^EE1-UnH=P;Oumo^tc)O0lpqiBx=u=Qg9{5M+hq zNx;J*svGEPIc!Z!J-}*cC?SlJr{(2dNhz0=NlVg+*vMh&c2609eCj4Kt9$>tGJHOi zozrx*TJ5(l08eoT`sEvMlR(cTW_)-x&sM;lKVu&&j_xAAr7Gv7={Jb{h=Ht<+ON$} z+kyASB-P79<~@;o)zRhDnZdV$aF*IlXsmq2ndXXk{g>f}$ik zi+7Z}ac$y~H^x7+J^GiwlOdDQseK5s``t85u=kcXNvH&y2fBG#El>21bVAak6g?6F z5RlHEqgRQ!XETH_K0nt#f<_2Zd(f-vznV@I@J>LZsQb?BrD{xC6STB`f3`*A?VQ1@ zdEaBKr83l`3VJz}8~l)D1igh-3^6Tmwkqh-$&3x@2F&72DOgk8d^9UpnT=68a{`%J z%{&`i?exq`(#>=1}4QxaH zoXt}(goU{*zqWB7LEd0bw}gFtNK$MuyFymE;@V$}*fXT(*1s_FKN6BFg-)V!9Y-Zw z**~wfld{Go{84>_%bEI^1#QV%#EY$4s~h`A#WHk;dZNVIJo$V|$+r3@IBUnvrsa5< zmuszS4p#uCw?J$x_33p(2K#zcxi}BY?zHdtj-E;O3K0QHFKSUP(Ts5Sf?tuu1Em$)e@@nc`YsFt{6%l_2q3+yBY0IXIunw)G@dv&$+g? zk6jZc(iA;kCUu}yRVyRNs;y*m)$`(s%W|y-lZ=7~DK3%OAD1zUv*Lcnu`vPbD-m82 z&;r}r5{7bOglxnjR8fmVmt((_DH$9OKBL!&b>P{1@3j6T?mUa|r6Zq@d#pMs=kR4* zoE3e;cVWQ!M7jB^#pXtnag4Ert^szM!rakUW;vDvI{)Mr2wDk|rr&>pdXqaDcVeW} zj1pANP1vP6#Z5`7YRs|RdJiY}qyCunGB(k^1WL0EC7z0apeV&4220r<1UGOjDJbE>nBnc*lzO&#E6Ji+<2X_JS-n}{6b3%M>0*&XOTg*2 zIP}n?*a326lIT={`%+lX_`+1NcK^LNjTDKhD(t2h;Ajc4CrV*dmKkBX98||6U+}v& z%Q!AyO%n%LjjrfJqk5Ht#}IZ(vN7~AALr-EG;VVv6_NE@N6g&k^c#!p1w{H~ZXo#2 zWhHbQg9F3U0zHNGFW+{%2fS$p$n70kj9P8-R{C?kZ@6qd|IFh4g_6lJs<0t!Kj}Ef z7eX>bFg5~yU%7nf<=6Zf2JyNN?mZ*FMt0DW!en+6)V&?fP4DIZf!&I-ILqEna5_?- z+v*ncYL(m79cb)N)0-<;bP4LnR>&Gqq7=34K3*A!oj4!wuwQp$BgBAswC$6@+8!gV zBVS9}pZSP_Zen+Sy8rakbiT?ywKQneFnDxx@n%hRGYnnadn$bbVafrQ+b17JlMS)2 z7Zn9H>}shhcY@1}!9Ua8Hb*hxEXx#~WUlHt9I2?b!|*8HaGQaT=tH7a7Nn4XR@ZYP zQJ)h5w>Oos*S2As{2!d5XsymN>+PWdZ;TR`J$w_OW?tIG&mMkeP+GoFK?q66D@E)H zDeiRf*JC+xy{Bo5DC9BgRG=SUEbimHW~IL+snq zZ*r6VamKT7qn5VNY?kxNH>0$x`>yRM*b9`x;0*^4)(;IL#W{T?crQO6z}5%qY{&`d zh1y9aPZlLDaX?G5m!acTul$pxytgE#8tv=cc5lL+!t&HnA=3v_ledGd&VKh_&pIBT z(rc*dmv$ux9HGRjBly%wV+?xX4VsgwGV-#kGt=}5acJvjUQZ54d-7&@MwN?6+Tl^` z@PJCEvm9rT0gT62C7hop6{Db;KSV`_p zqhu@UaO#~;oq<{Xv-BR$n_(O|rN@NSYcDv+1Oap_x))FlBX$G4a`&+?IwFJAwE&_E zivgzUFT<&B%xsI#8~jWQf>cAm_fvsGsAWGfDlQ+&^`1pSp}yi*L%KK@iLH*;H)eM_ zU22nL{0nXss#$e(0{3A{`&9Q>x`&RRsrk_pFkGDUAdk)sSOwl*9`+2iJi?(vJ>5>@ z|DMvTE7g7T&ARS%&_I<_KnrAdz=M*quwf%}}G zat=*JHItr%mmk%#UZdX#P2Nz1Ta^NPB z&`6Dp#i5r(chrdMfoefwKY^__y7v^p9TfY`)k1hn$KB|K)hizC31R)>b~>`qrstn9 zQg+==R>G@4O3&MM+@COLuDm)3k8tzB}k|q@PBv3kPXM(^ji?D_f$v-mU|!&aMqRG6^&{ z)%u-(FA0w*mlDZ*md<5))C+yFVfTt(rFD;P@cM#t?T)Er?fwzebC~D8FY$D=L;9g3 zVbV57Dq|MsVHQg1`7{o}FK6!*kvi^=`^6mAkgB1KacJy%8ujG0*M&dQ59Y&G!JGX! zC?3IBP?w`hMPaHd^y`bvlOm_e8a~<659(hzWUf0Pmx8hK2$uupogj`9!^6?ll1a6z zAjP^&0S)!c>@g>|$kS&=FXYn(s5vldRnxxnNA$FR)lt3LA`W}CQ#6pt_fBX^YR~$R ztc$rL)c}$P+nL{9Jxmks$JyP&lxkBX@@%6FSR2uQ@7>9vicND%CTRuW$1M zCPSFl@(Zr5kKcYI*kLsutpED){#zR_5S>1)(jTF>I9l(aomzJsgGCVH+QsNWQVG%&%Syt91|U z<~U|&Hk=;sX0UvhQAJ;OF~%c^9Fl!Oq+Cn${--&HZ9-UA;IaPh6>+zwAH?3?yXKYP zm}7jZIAf1e-@At)&C^Rl{Ojdz^)I&O4c<*LM%9yyX{x^B`#|9^;W$&+n90Aqx)SjI z)r%#{DUBgP_YZGZm%|@2ic}Z6&U!AqHZt;Seo`HWkDldtB$?pNA1iXWkN`P6q0?@5 zD)SmN{KoNpaGS#|QYQBEz4mn@x^7=D~9oSlMF=oV8K6^vXR$)&>9Q zQh6@qAM@+mHLTG$8_LZwZM8}9DmQ~Xdv7E~ec`~%}tJy8T zYpQy9tfFop&c9Z6{z470woNgT!~>_FEp3%UN$y2g_mUBA1suBPwUT)MaL6!RGA>ql zZp*vuC*Sq*VJX=z+*izJu#oO@KhueBBMX(D9{+4gipV?_NrEG%-M986P@;j4wN^#f z;P%G5`YGc+l$85yag#8gnNcVF=TK_Q+}Veu;mKDQ0Iu0-_z=37&TLL8e|Q*K0H~#Nw!4eUJKpl*gdJ$AmxzcXgCA;r$KWLS#ct4nZ5T8& zrRtIn<4!5`;PuNVG19wlAFj^NaQ<9Tz-#CXYa|_rgnWuKW?J(57I;t9^;Hxi)G;ag z*K64kxo7bvyZ32i9BwBh{xOs_Q9EQ9h-nc?C%o@&ypOAF*LJR)NJ&{9|1S#TulVN^ zB;==y()Fk<5A2 z-Tz&q-=YxNRJ0kK8fFkkpVeF>d*k2X{(o2P7j5QLf);||7KH8o;2&S@zvA>103>@E zI?OL9|L0S701}-k;@BL9Y$4-wr2n%B1r1Vaz~wK%(P|#b?*F1se!YpE4vEefTN(kP zU^gLl$NWE}7#N^FZ)9R9#7CV#{y*BPEru)xRA9g&RCdM>eZdbQeE413{r|rh9&QH4 zKfK&>CH{%X@x%08lrYk+;i^UccQF4?XuC;lD!4q`|*{Bt7z5JiovE7ynX_PJm_iZ(l%6NZ_+SQn3Fw z@%lesVX$DzFZwgIAN31HLyYwK%|B27sCiIKP+;Gm>HW0-pmQ7u{hf&Wcgr3!<(vH} z4$&q2lNpAW0DBd{LeIGSY^y-{>!vN8ry|`U-AE|i-QC?KARygcA|--!$I{*1%`Va`OD#+MSI=|K z^PKaZ_s0jm49w2#%sn&rb=}v6_CZR@AcYp2T)A;lj0eu}0Q0DKPji8;*E}MAfXq z_t2UBE7l|NfaLr+cJ-){Yr@!g?_SrM^he0S6$}> zxB_^q?k5z=#M3B;1T~}^296HF?L%^c#1rPa#6?LT^CK{M$@2YIbfQv!P+5#_zVm^k zrwp!$uyE(ezfsH&c#Jil&{eK%1C*pNx^?JlTqlA*{f{J zCX$Iiv3@=ldlH2~A~g2<<5NBg>>`O#A~gTzD_qWlPPPa_!R2trJe*jb;qL#Wrlr#! zcUlqDNjz#4^hAwc^@}x@q{$uFkf|AQeI{yL=WpWLP6rdH`r|p#yaQh?J@~vp0b{AH z5%JI1U(5y+G5osxZcfr2Ry~Km{1vYEhMlAbOS1>R}82?BTNe zRc#rDn3Np4)@1k$seYhuWXjRr**#zQwMy?m86bNPZFzWnxeIiK3qAOyu=PKt_zCiBZM&9av!eTb4&@*!Q z%ID6Mk+cz-LBeql#g-`syymi86I~Rki#zG zI;UNyJI*(n>j3V)n;y(dx<{59MN*(hg6kIDMIEr)%-xE^Nkv~7_K^VZ4NDcWX?Pba ztSJ zE%DA-O3!MVONUpn4(c2g#&>cOQq>XKC8z4U{?L+zCB)t z>uPuJ3n`!5L^@#0z7em_#4N#YhW#&YZc3q+Jz#-Ro94P0?enY8a80*N=wpFj%wuxC zg*_WlprTLA&GGzXOyNzuCzK+LK>&kO`{VcN!2;1I%aP;ArIue^D;LTpRB3CKPe85B?3S$w)Weitn;kNgWy)o( z4AiHdbf|+9ExV^!hdf@Q)rwmUGSp|gN&rq@IsY=JJ!){PA81)?T)&PvMeWOd!S6v4 zq$0Z}qI(H2GLv41?=P2yIY4o^(6zN4bO(aH*NeT8Xglx7tueMnG+}Ae*sV7$h)vV><5J1WZEO#&@9k_a%NZ0tj$5`g3Ql@;zhw^6K~rm&jWvZ zDqQ&P2ZXv!uFF5%Ve>4irB@2moKvV(uDzGufNM1?SSzhIou$Z7_>H_HWY)J2#D+WU z#ls#|}T$B0#KLS71zUlX1i{?wrECP+FZUmY;} z%-8g)q5kp&l50MmB?Y)^&4z2BjkfQ1n8QyFfPaJYP0!^)okEA`Qx;?KCj%CKSM&5J z{L!p-XGMQ&(6t^(Y_VKw%BD;_p2!l9!;JQ+gYssdObi*dmXSdZ(G-M`IjQq0&vqx7 zR)K&dk=q?!C4aC;rWGAT!mjSvS`&4^wOWJ0yQT`QaTDd5t(rQnmrwzlD0QnIL~|RR~lS+!}iCPFp@_j2M{qk#J=N>r-B8VcyQvni+{4& z&6coO%DbnndG}sjoOrEoD@QGRK7D}gTbw5t0fFgxxFkd#PIgp-@%afBqgq)SH!7X+ zshk$~`(Yu1fTzf*CAhwpZ*5?&w9UiRu5`MNv~I0?rqEn#OLy47GW?K{_uro&ia?EZ zF%3Lk0)M-8VP%NETs_|$KXfk$L5}GjZg2jfIZkxE+UPgHS1SN`Dw`|!Di^wwWhyuh zDw$wkgzGRGp5b{+u6t$O+7dO>hRhySb$mX$Cerli6k~jpp-SU!KdA8C+fGXoH4-UQ z^EU|DH1@vB)Y^mFHo9s(2*v~}4tCrA#?u11?N89SyTZkqKItMjU4L18el>6Cr*8tX z^@O|cjCpYRjBwWy-`!<9N}x`7Z3fcPL=kdfy*&(Pz;N9mLp`LJeR>qsvc^zL4^dB5 zhQiko%^I}Zu3b5?7A5qp)QeHSC%@gKiP)3fk<{`q=>`SCUTT@REa+qkLm^w68O{T? zr_qjZ_T5BFCn#$s>5U{x0a8-jSHIK9>6UEgd?vNllxBlu_zfnlG(T`{ZBhHNC)$Z9 z!ZI3!{S`uy%%r6_oF|-p?sT^CfZ^Y_`T5Rk!*Zr+T~{xPfQ8v@pQY=;I&-*CzsWC{ z;WCfUNEFk#lg>|v`yaGyQ(Rg68sXWvzweAKu`RebrUZ4hBupj-;XyY0<)*0jWn$XQ z_x=aV`JlZYU{EkS<6(~ZNQyXI-H-{s2yuAxxY%vU6=WBfGZm7sztV=_Vb;yFhvfNQ za1=~m%|t7iBKsG*5UuGyNSoD98dYom)M`V(o{;;3U8@glMHw(1PNLaMj9|OO+vyRo zRLrLjMZHRc4}n$G{fHSZ#F^>?ATpnOA{IFX*AhtyPU(bEz&p{aGeL(Vx1}>G;w`hGw@GG)4tcbxYk5QAEjwyYTL)r$Oc8e7c{By2p_>Ib# zF=_mK8UMN+r0sdU#DNk%lEy;)L&4SES5~*)8c4G{_`s;))GB_vj!bZQ1PS#S-}%LBL~w|ci#$fRZ0EFgo1v92 z9Z$lT((Q1=OgD{Yal0*J^kOwv(69D2ljav4NArtjx#687jAbv*9089gaphVML}Jda z&z7+Kz!{Yhvwr`nEt8&g@tgI{kwISuPqP}M(FV43!m)t z+jcp{S|+Erv6EW>{~3onT%$A`zG>0Py(aAF^%^T&OBaKk2gyB(y}{q@7jV4LknI-P zy_E_>){;B=N&Y^_ai-DqZyVx&JKFBUUr}41WVW`tdr?A|4XH(w&G5^us`85qzbg%FO0IvYD5J)N07 zvNJ&`;s?)Z_SDN%k=0@co@uGJ{ItqNVg5F)T0G*x8QAk~w`g&S6Bk2{5Cplh48>foY}Evr;IHHBT7R zEX>ZoKWO^IyUF2Q!T|^-8zIp0%L9mQ;c;C_?3FL*J5#G0rS)w`z zwMA`3VvO9^FGK}z;5Q9dO%4~pCzWr?HhA$NDidgfmp6N5mg#+5+ARjP)+JNK68!mr z*A1Cghh(`5bK(nczz@#hb86vfx3)+}fw$9yJWPfK{Ykr};K`A`9BDF+D#Hg_J`yuP zviO*`;dgHi2K67Gr|nAM)+c>krK|eHyu^IVcJaI7s=u*K?2ZuYK3I<;FRw4M-lD%U8Ed@M{n}bD^unBgcXcj zEs`L49yOAyR_YU*_+I`NEzgs+WAs0k6quvNO;*@_74gcSjIjv!N2_Nfe8P!{qby>Qve+6xj2#~7TP29soJF* z_y%k{UCWZAxNw7c#OhuZSsZ!x0KlnV!d|AMZEgl$HFymZu+^Sf&7w6QlrVw{ogBlf zv!~z>8bruCsC=J0LHeFYR=2_4@~*?jE#}RcCYv_JV6TSNDGAyDUdAT)L1}!c<@@;j z+{mH^C$m0(NBQGP&pN06`dF@D>2N7-uPj_w*1YdrT!hgGP%+g7v+2SiWUqFkfR6Ek$zYx5>SJTyGB z_5wJPr6t(h@KnHaGR`PsH24k6g(h9z-yziGp^fr0byfLtl2YIytx=~*CJo7%L7Tr# za^Uuy4OiHmWn|UlEeDlJP0_TB6#`79&hmZTj{I25?ho}cKvvQM?BPWY?Vf#T61fX72K=wsVt zCXHPBFLo1urL{~+RAc?Cka0}dD7fAlnsNRf^HJtm<>%RtlKZp8S=pUWBrNW%V8Dk` z+}(F=qhS+brz-_!pi%_Mac7?=m)W3G*S}GDmk$gOi=bOxs~m1|BP1%NzKRgK&qK&{ zi%BNfrsySq6C{5>+Ri#x>P5Kw*;Geg9sZ;U@Bd{07ZyZrqw|Y?|BuVd-^}9&rFc(8 zcv>k3t=?``7w}M7y0-cw)wzE(t-(4_l;U^9r#gZY^R?!W1JhFttMRIh_BJ|go}upI zLPYVYxJ0HkOX>Zdo%cV>p|aFooI+r4QZ;F_Vj(ircOpZ2z0J1sj3xfy9pJ?VEI-p* zLJ=;9SFJ+>>_pRR=L0>g6w9TPs`$VEoVe07mjYL#i++C#o#2J!_EkmBS{K!9Ht0Fk z?KqCMWMy%fNf)>b;adP6yKaJQ?gQXdr#}fS$aX(GF(q3?v6*G-ACZ9zII^Q`7p91Z zdum$3zH-`!+_Im7y}z9GxV_}uG;;DclH0y^6j83r*#q|tfxS&7KpHKN0-?yAh1GV8 zhS2j+FN?dFpjc{?u_g4LR0r6(;-mBME?5@f?E1#}h}p1qtlelFJPqe0J^kJcBB=C+ z^d=azUA?O0bGzbX%{&7@esZi(`doh8ZiH5{S4z$oZp+^{-))?Nt@;H-;P9-rj}V7T zJRPn3b+95IAgL$SNB}p-BFBqFhhIz2h$`O8O zN-jpq&(O50e8}MxNziQfY^{bMBx=&G3P7b%zCoqrX>HdyxcMYzx2sOQu`uK@()rLR z0K$I@DBoyY=Ta?@;8!~ad+&|&8ih!neNk_NdTvK3kbc9SAm(#zilehN8j^K09N_Dq`_JI-scJ9Nyc|d0d8$<%Ta%*Byec` z4df+0BNcEvWjAL!bI(%&G?Xa0A~X|x%xP~e>H`E?#cl*_7SoY+EfSCY*n}r9xhO|y zsJ)ul-IF@<`5{Eu9&G2{rgw*6@3BGY(Dh^nnLpOcui@eOPbOGMaD^A1QahL2%4<7I z*zUYe-Vp9g+~@!#iqZ}EZ+EEl-d(J+1r%GwPh6}wzUohP`sBF5CVUF zzU#u%h0noMM-mg;jpUcH({{K?+Zj(rZvc3bABZJpY;bB>-h*6v>|=-Z-F9FeW{OyZ zh)-OeG%fPn$3#fNn@N11B!5*olk~LE+BCGWeq^y2_p}#}-m5^YRjng!h3o@0Kq_fL z9OB;8vRe75WuT$YQ)$GQ&BL60qNg={*AQ(n{u3Gw93ZVqCtuaf7HZ_O*^#Xq&j>qf zfA~ozP%X7ksY8!??Z83XOhB zp|_-%@w>WLjH6s<(BrL2>3?aJ;WY#eK)TSTL(}rg1WEFXW(-^}VaI{P_TiUoXL-~c zr?g3tfc$a&S9e~Zd=#$hV(D1h5(P(6BX*~s_8ZM>G0}0z1UOa~T&L1xM`d_zz|Q%X z@*P%rfj!mJRQpRd=Uky~($E-)AHEE>Q@WXqCz??Q)h5f-u=1_W`pydxOz(w*%__lu z=nwPZa`T0lk0oqmq@y)OVe$J)>Rpc(&Xy@%yb-bIXi^>EpzSUCe}9 zr?tK>{BLgF{Fv>7T@DLN$e;oB{Er)Ch}%JY1cwZ(7MmCQlU6w%bAoZH7WSin2?-s- zP@RmlL5R z5;*yF3yZ46=V2PR-hP;Q`(Z0`kdmXywVG|hj?-)a1GF(HaPNrR+0b(<$0ouOd_xo% zlN_r|NF)$>TT9`k&z`?XxBuDYnx@(S1Zf~#*d~%#PaZD+Thk}@?vJdrY##YZHXMQN zllY8+9~<0-SosJV$?P?Uh$q?5Hp_kPctg2C1gHc_GB6^^H)| z>VGsNqK}rQ(c)ZR*+vq@?CI@+V6AMmPd>}7>)gBY7V6FZUH0XL3|-?whFQm4!Xf!;!ocT%|^qIHbku16PUM~Fo3uJ-x}qQyInm0fgU z-W)EHQpXdO!6l=(ebosCW1~MQX=ks|_yCmKs3qPBc!HvjZz*6{i=V2e1~`;xkmE+0 zZ#T1HW;u~Vz4r0)nCm2f(gD+&al%~x)fqqe--h0LG$n6|7Q;1 zzfAwE#H9@DxW80@VN!&Xj{hYE{C(j$)E3dug{ZsqMfLZT?ft|L--_@Xs%1{$~z`atKdS z-zMneP6hsdTfj@i;qa@wsfTuySP#yW?tcMco8hzc+)JNZ3s*J|&ARG;(f|L!1L4?@ zESf%ifS3`UpPu^7irIemmi<9jzrH4)3jD$P^IwJ5|17!w(1S)1_ zIZKYaGp3;E`kYf2a9u!}y>PO1{zViO#j}o+>Tf>_JBku8-m}U-3W#JvgsMG78 zjNHkZ7TKg-|4Sj@{|t`_2LUi4=I1_Ag3RD}$uJXqewGI9!Pa?Kx;z8;U+UNYso^ie zfERl^3b#{RU<`0Xz1|ngX!QwJXH*^M=#zUjFa3X`#{c$%K#2&g#U}{F?oLlce9JHq z?0nrWGa)tTWtsBt+xuT%fvmvu5rUtrYHCY%7!19^TN%475ni%+6;fpUB!98+f3^Hq zFRa3d`)mQpgbn>)ovGdd(VBXLyW%z970dsd*2Di9N*^l1N^!l?t{UBpPjXK%&Vkw7 z0dKUVf`;7g4iLOXRYMO>=yh-O9e} z{Fe}=FZ5i!_ASm|0Sn_-n#V>AXO6e|G2;T7FE+Gg9m@l<-wb39a$Y=FpDp??b0v;D zg(yWu11B>)A7ri+s#e`ty~@j}@-$>rub?V(Elvt0#v=LCJvFuuxyCNJu*YG2!j7H` zur{^UZm{xJGdlUH3xHH>-u)^P<^`QRT=H^fFY;6{6YyK4I{awt-w~D~TE>Kt$ks zWt$w9*b8fT>Tl00ULYP;{$b)oB#j{B2ys!mBE&o?k<8<5rWk3JV;+BibR0Q9LH$Ic zq|DxVaPO<(MTw~iyKbPE*x&Q_&&U1TwflRxaK^%oa9EimbjT+$7L}J|#oxIu!9L4k z?+x2gwIko<=Ff&1$f}{pZYNNfrFi&6L4KaVOPKy<^ZIjSguuxfKrL(Dj@3@93bP-? zRJbPHKnj~Fcx%vk%t+Y*t??GhQ?>`x-VR-BL|K^+>>arFYVeMvM5^uRa0MRAU{j{P zKxQ1-KMSC!-I)rMv^YMmdE-;h&WZaQqG7aL>ln>!RIM+4NlC1do5O9IG?pvCrV_AH z=N5n8bl>@5xH%_v@s~bZTjPnMJ0UA7)?hqEdSz}aP3u`CZUWBzo&UUyr{gMd{jwUJ z;_&?eBs9(FI7BpDDJ6*RDf_o&PYVdIkoTn(YYN_=V&CCDHl=5}Bq~<_NIVlY*_zFz zr%$Jy{`VvWe$4x`4Va;IADtyD!Bd+yiofVo3~Vnl`9g1Vdj>LV*aH)_8J~Gy5>u-1 z7E>Dbd&zL`S{L2z$4WHl2$i_1B19tn$S8b$+Jn2%Gdc1rfn@^`=q?ZML78@ zC4~G(Xy-d$;OcXoG{=@45ntU#r&U%z&$GmiwwpU#h_*DvMQvP@Z%(%W(ZgoYb;t1v zVDy~FVf}-y(ArvG%d6KDjDoR?`aZ57V^`kGcQh+XI}$OO-4~Ew&$7)_D9kp0xEtwB zJtk+-oGo)c%LII6nNZx54*9q(hS+X zkKJ~X4nclbVoJ1lfBivUwC?dyD~13WESPrhQ96PmIE)2FB!`$ve9Gy zVf^rx!)^OZSRA>CmN0l=cV_$7wQ8%W4Rp6DQbJg$`Rv=hh;{v%_7RHD>4OyzsyI3P z3-f4Xk0aUQDBJ$@l$ZZhrcu0ie6&YpTh+4$Nx+$>VFl=-r>_ra1hQ15ebChE!Vn4GwaolwR3g} z(Xkk{>l+5{;FszC%&K=<7AF|fF^;mizvuC8=C+ytfJH9!4!-6S*Hmj7l^?5+g5C?s z#FHgAOsrpw^-gAq)G2NH-~alM8~X}C(EaE#X5S`0?q@OPWCk3!+T?!t zeR~K4T|EbkfQJS1M;sP6S@%rt;hYuM^DgmB;ebZ6KL6?frgDc_Pu0#9gA22CN(C(j zKe0)KTB}8hWUT$Z8Wk+qDzvcb;phEYP=@n9$|xUU%4&z3*EU}|v$idWQ#=3RIsdDa z?#G}F@Jo}nUnKQuVCun_Pmj6RRhfeun>Rcsj_dUrpvgS8s`{Y*t-b{9KCOZ8EzvRh zr`QlLaInq&yw0+C8vcBS0#CEZ#9-v{R4&tpwqaV%;b>LJ zXM1zz@|a}l)R?wGO2)a$E%f{S8*thIDjuLw8j9IgZ?8r8YUz=)U}NkXxnARi7 zRr>wq$+K;umR`^zWzhS8{iW;El~G&JN;db75_G%p)G{p2AtQuc$t)r+ndPXd)m6=D z&HRb~g3Ux&bEU>7qWdfMo!%R(2~jumho4uL^9g*;?cB;sRe_tAybt^O&T^bbT=_x- z$w+?pE>!U}7jYEay5a7WwgZAkRXUdU%tqXHiI?{k>BqB{F?^ff(lXNaiiaBr7nAc+`_o0bX<3CKNe2r@Ww?jm9C}sv>ZJ6IMp!a zI=Z!(ZohEZAv`AG^SDTfxwk1I7VVuaoy-u)7xB;yq`YnGP8X2#VZII6vfd8x)~+(j z;BJ-Ak)7bk?%`AP^Eyt&wT1~9Wjk*h;D6{WGdjlO*nZ#pa}KkTn@jP#J zV3CRi;g`kfM?Um;}2`;Wr-i&d`pGDF{mu45*RHosKJFx{ z3Pwp{@#86Q+et_0B$7Pc(8UZ3cROuwksLX_-(I|!EDGA|meG|3k}{F@KAwp_rqx+Z zW`W)20rD*~wKfm5yFz?{$6sDEs>%kEUS^MF@yO9Wap<7j5HnwqZZLQp&U1h}2cBpK z<0yEqpi};*`YDmK2QC_Aui!NYz6HUfivqQXMkJsnO@Kn4{Z)!Q8JEg!wLJ>O-sP3d z=1>l?+g@XvV{(`q!CkZG$r24E09tUcIa}2?v@;QURBp(YS!bTs4uxd3U@TbWfBXEk zKO`=WjNdIoOUKaf&||sXq>hZ=zc1!wRSA$BlgUYf1WJTaq^>)E6}r5Yv;_Ph1Scl4 z1_=02C>D{!d5U#a!qoD!E_HtU6a}FnWW;oCJd}WrMDXWz=b=H zaiQz8Y0;7@B>=6@1u#+T@uv1bDc;HV`FQ&0PK{Ab;*O$6&{(dTK+qI1=VQW)RHh6U zJULA$vA}r32|Gp+V%W=}#@N*_*v0--Adg0$bppN&tW4BNeW>fOQRfql)Th@xOp5)V z2)IyuzLAyKRG)8mZaiGQ(~Gko<5VZ|xH;KyigB@I&a6{^@p_m|u<Z4KoOltoxz!G5+cjucF7w4Ss3F?|dJlBfnr+(y ztvV4@hS#ublkQLCCU;6du%yg_mLtrW*q1)TP>&M6+RU%;egsHB$B-^+s0S_jds0=Q zprfJ>WjpNmB_E|L&Km+-Q}@_%i~=9PyoF{=gujwbvYX@}gV~v>d?G(q0m`hAD3q}@ zyaLT>CNk~I(KF$X;vsU9ddCFarhGclALzWNXW0o z^Q1K#n|xaM-elKsq7wA0I*|v)cM3A&1`0=` z!sBOYG3I(?W?O6MDTI2&(>nI*H9$9m)O3^4i8t$uUm+S2P)0gP?KK2m4b(eq6bnX z+00KCdu)8SF~)n2_8E zf!?9jl3$yR?`Z*Kt<$;3A5ta@KDfty3-e2)(0Mu$;P zGuh3>_8Miwao2{T*&|A#mV7UmyyX+Go?+lk@wZwi6|Z9-UrV`4fDiEiFkKj6p3Lca z%ZGQ-r}cj0;n<+eO~!#1xJucXXk`>M(1*RVUA*w?JrHiWtHfzB%sfiS!V5jptl~1+ zDm#!=z&uN4^I(^<+xl|A^R4{8&OB;Z)bluFxm%@8m3&v0T^@n1GH}+qqpCk(W={BW z=AXHMgYuj0uRgPbiPTz(2QyE8W!?c&I^0e*!$uAMsm_nJ=HrNi65@FAO8K!kXJsC3 z3l|<|6Bh=XP6`v7&UqPVihY@G!`BHZ_sAE+a5~?+pzeE4w-I^B`PjX)x&3ne{P8oN z(<>_=1E0hO5}*RMcqlkw3US0Ls`oxD5=td!#|QK=c| z3_V#PH+{p0-&({6smwexiSzshVJ661RE^u^@?Du~Llp69kxS4EuOa7%!2lMLY}Y2W z1u~uSPzFp~0j$ba5Mu!c02#%nCP_*mD67%otx`2lHzOnb>h4!zV-ln=rz4pLl!D6v##6T&3 zZx*$h7>+bKEqV)#AkOD@q0+coGnK4<4d#=n2DJKZ+ujJjIB$#ZX)XsW;jRf z7dx=ReX`tbV!HEa^eiFlxtJ5LNVX%K*MI5P&j^t@>9&3RonyKVzb0nfUB70TeuXSU zHHtKR@uB;+!G#nhdHmX`qp1>pt&4ZDokA-_NTYQiF@gPI=e)I@E009IPfFr((Dk|O zmx*x!pY1L8%xj)XyY_y;N~u41aAEIhr#qC2oarCWsQTsmt3u~4zhXehY9#67bytnF z88c~4TQ)h-5)}3lD@R4Un3Bt!Ic70=;J|Ivpoz$!4f=F!fNYE}gZIda5F+Q|h0fZ) zzW(Ah8_R2(mkwo3a|Vx$y4DO~-}`?x+a>Oto}RvXd;kF{(g*#9UuJy?9$Viytq2hG zIYt~dojmG+)w*<2C)Pg)Eo zm5nhRrouV^B0i?}0U`kj2??<D_Ec?LTdsE64?JG4A@tWgv&;&obkcE%+*Qp-H&pfV`L z6oX)Y_C`>5@^3y(=o;+@!W;GHoP!e>`bZU|1QkcP3EOX~_PS4kf`eK4oWj>qhN21C zM))}+Hl;DeO0@(=E|6ILtkugnnu|=ws%xxgS|eZ2U;T3Yy|Js|g9{o9fatQA)27Q& zzoK&)FLBk(8+dNH#f&zIE)m2Iuj1*Id&Bs&$D~tP0~jRnva1lF&anClr}b(FY1*yEulU_^tCsg8Sh&U4iS#E#Ageg3~t|7eAEE9RmGIb zEnnh@_@JB)t5-%DK8=x|oG4~D5T#zozBahvUu7kvK=svOXEgc#T&^-m3oXuMl-?zX6$_2D z(1{5bwb195rsf=srp%{v=1D*F^0xtXGTt(*DH@4aN-7 zw8WRMdI4X#KFb%XjuLu~sD8~{=iw#$eTDOyt&$#CGj`0d`%x(^=#>p;kp&MgG73sm zwv$caJ3rN#z2T{E3P|Q$;bdOnQ_p~7TFYsYI@QKuAT+<)GX+L{|AT!DJ}bI|GO>xa z?sxQ;(BAvSSd%z2kLyc*9J%ZKKAqvMo^Es8+QIk{*;e_31XeX&c1paHi(~LER|NPP z^1hE=vz->s88x&NA#P-TsXjt{UDP-Hs;zK`u9?Lt2E&Ot(=oZq?{oOmQG~8dGh1Sh#;%*2#?Qio7Ez>ye zp`&5!F-SyF-jSM_Iq0Mjg~N9|;Kg?@DWM25ZxOGkG=`8+>6&7dREsZ*o=!_s)H?@p z<9+OMN;z+YpG-MS=yfiN3Z*u>(q;+*j}1$sr11ZGnzimD6?Mfp3XxL?BQ5 zwFzbu>&1n2*-ToCjOM+vr0D&g`p(iY4%ROwGyf%37UV>_8T?`-^(JxXH^GTN?KQ$me*;EiaW=(C*oT%R`B(L$;8uhn(JpC-g>I$ z(VafyLOsAm{9Yopd{Mv?b$8=&kEQ*lzb#3crXq0mP8xkJR6GDRk{98I%J?lNV*lvt z!L3J0F|D9VA1k4B`IvjRcUet#Sb#Z~<-RTSpb*E0MsSE9LJT?FV0Dz|(+4Ocki%ie zU7muASh(|q#-Vbx8LRv}dZAL6&fmSE(;pi3W&Er&=YiijKRgPVslelbhltA^L!f<* zViy$C&y~C4J3is^-NpFuHYf&c${Bb!1H=>q{gTcELNg~OsT5l0W(W;eKNLRi;5ls^ zx8JNNs{=z8J7p8`%QaHCurGGZ5aGdzOJ>pNAgb7+Y&7ndAkHFvf6P^Bj4J*o>R^sT zWh^Luw8-X-g{FFy*7^zh#XQU?lxU7^vy1+`SEbroe7m4W-k)IlXnV$@Lc11mk|&TL zv(~(43mo)T5`5@LM_}OlomG+Y-e~dvp6cH+#GSXc)OX;!eAWB9N#1b`vitkNO#X=yU$Ywxo_v=a^a|C^+Gh4D?PiJE zwv`~so{8RIVN_tPVRY-xUC`aFO^c`T+lNLDmsB@3Zz1eUtZ%Or_zPTqC=Z9A@eG8( zQHt-%ks1szgBQ$`(T2s%4DBI#NliFZ{Fe{k}(czmftT zMlipLe;y#4(06)tN~S-^UUWq1AhxIZl<~m#g*UA-aPt{VTBmHe2tIa%xTzKQ9p9{wI^LAT z=C9*3W68H`r3e)SbSlV=~Bxyb#JvxxbkjDG5vVJhAt&`+J6D?jHu5fI1)0A zil7yG{Hmg(K~QVtrNsy1ENBfpo)9VC7*%IBlzvx4PWnkQL&JE6@9m4}?$<`D$)X2$^JuOTi>M^;3+NE9yE7Kxx)^=#kL#$?M)Tu`MujFq z4uj)DV`lT0pqt?*cK0oI$at_#IQ zFXubB!nM`3y1iPd0*fDl@-j`hpVvuhrY(uo544~gaMRc7an+sN2=0TKt<>wP&t`gq z^ppIdZX4a|C91^E{Jk^Ca?_R5b>F+1Y&S>7_mzlU#=WPS`#M-#8;?*}m#v?6+_qFh zOpBu;af9Vv{r)Ick;i3o61yEH_n}z^0pbk(@KADfZhk-~<>?!+NV-NCS{KO81C6qh z+@H*fN3J7er~g1rz>5zYNUx}p?jU(JdA-Mypw)w!=Sa1Gj^!>_l5;_mkesFWNot(8ETx z$koLl_~T63*H;gJQ11eX2d+j9BjYUxj_mpzvdf`pSzW~R{IVI)n=|Xiw+jp zU)5f;7a)!kKZ+P#rixgM<)Qa2nuCuf1LM5St~Gdg?MD+>kbWdA+N&bTS+XEoVRM;} zKlA+QosO|!roOwDe2xYG#gizU=r z@OR@8mTZh6I$w8^whZiVnceShT=iilCWakex7c5C*0$(YP5O=k$UI*d)V5ejRw?;~ z9s%Juak`KC0@ZG#te21V*v-SVnA%BOyT&@28Lwe8i`px<(*|pc!eWxRz>XWM0Cuxu z442L(`KM-oli`YDB4hLH{F-RH7-~ofTvzT8os(-!(iclS_!YF<;@KWCmotWSS2k}y zjhzX};ciHHYFC$7RoV7v6FzOc#fqHSV(>aDvYg}R1M_O;qw8Yy(NwZ*W2{gA%mv8T z^TS&Ab6tDEH|=am#q!}cg{1}dv9+}|6hNf!J8vwY(6w-p=>U!c33nTFB?)<2lF zW@%ogNKWoYb0y<>nMxLRZRMjDo)JFub!im2;}Q`WKf$X{nVZLq!4))b+wvTm&TK|G zpDks|XK4A1yM-AdtkG>%83ug}j7uFSI9-0-bq(Fw7F1Y?u@S39WQMelF;?ifS|)N~ zr&+u(=@Q3{4O^!al$&<#ul7H6T_(Hiszjpy+5xavs5Lcg_5&$^Gcn-cT}L&?H${#V%BEXR2d#CKp{z=hw4r@ub-4kB<bZLbwGtQEr#pgNC)}DM7z- z>8@%{#(Imf%VtXuN6#!{-tO(L1Q~Ma56}-|sEMGouG&~EZ2|MA7|RV?iuji=jVHs;P~ zZ8HN9Z#dcH`cTLCeord7Na-an^D}V3L+@gZ?-`^2ER59K4ubsTX3|W#8NYR-UdmU1v-GNFjJZ}QsE@L!B}TwR|nN1zK@-u zS*&_!XzjgQmut)37cj3=I8|yQUU-Wr+T=E_5uhz#4L=8y)~F*5uG!8s2@i!@^HxR< z5m-*{&t`9-tf#zF+p*W*{5&2_q2Rf?*f3tH!`I%x+4OD?@T+aUKJ+Nei5QejzpUw+ z-9BkSHKlAvhpES_{i#~Hmn~J93qKqroqw?$+1?y-sj{6>ri~L?UGPPQV>QB%Hp=Nz_atq2U_)T**D<+N%x&@DOE_V0w)WcGeFF3wOJ9891$LS>N z=I?5Btr+UFqN~#Lv_UtW|K<^3!*qjo9v+E@3i5iDF6!>n#Fcllp*S#E%baM!h%!m) zvki(O6OHrn-W}wGkn-<%vtvg-82gJ9CgAQ8GqIKL7+t+jWU__J!|E;|aUQ(K9Emr#%cvJSed?29kDrGL722cZRP^#xn@m z0?!fY33;yF3$R(qn0IX85;|G1tu8RZlS@20*$h6uC z$u6`>;4=k^d|g?#k0My78y0Ob3mH;`_yd-}LTVs~H6H~@QVuI4Vx{H%3eiz3--@N0 zspS~DSh+k6=S9fleXyoU+h5EzD#3DsjwWiLF;CR(93HhLl<}QIzn!fQ5V9 zM3CP#v73N|gw=Bh(O!P zo2T*Zuv5J?jcM?5PJ<5{s5xcCn`*X8*s2%8zM$XzK9b`xJ;|Pju@Yn@)f&io)@xDEna)r^;fl67 zsUxXPb9;gcf`Cl;6{V_5E3R#Bn@{16+-(r4))l#nDY(H~J;GMcS0O~Ciq0eqnnvZ! zfdY`$7`+6)DYWZmtDuh#OYT z4oK#?{8<`ydH&_GlD8b$4$lC`jK3P4-^y~_opzV}&yjo21Ly-4c!h14y2E&Qu-cYs z?`*x*jhvKg7Z)mzEi{!_;{%RGZfa7sY3oL*YYR>6Y=U=~@GXV$+aTXa%M}7$QGK|x z%j5#HZ|Wvl0|LDxNC(!nR`>lal@svrWZ)aHHOYNQG~>Ou^f%TrZI$k-$)WRJDa!1Q$V*Lp$XU21 zeaxYIUoer)`|YPTGeaQ%(PTfpqEgs>{g}mO=<;DHkSa*$qQ;BajL*m`!vxU%vCzyB z?vYDd=H1J~ZQDjaGV_^SlUgy}l7Y88Vj-OOYyJZ}=`U-DmVxx0Ke_B@ST^q&T;sX> zetQEle`RARuCf%&o{LPw$NNrO`d_%}!6%C6*dOqJFzPsUN`$xeB$zyXUSs#7y7ZS` zgyr8mo@F|}hs$er!!0PEEBbyjDCx}@?9K0Iz9c_#9}C1bIbL4#qMWu{)1Y&P9OfO( zX3q5}%h3)C2j>lWOEWL*ZHt!=&ROgjCi~1Apeo|ueA7?gnT}y&najWzHMSd+T6d49uFcFWmaC3pwNjZQNn(5@-_2L>@qVk+@STE*>83Ubei|$+S{dXlr{|$OC0RO-fUcLT_c>c(eIvE3V#b@w9VeA4+NGf61&F7tx=*+fVy-DPM|M z7rdRc1R>x^Im8mYJ@3)!Pvce?&63B4q+`?tZvqT1M#2}LF^gu+#UHVFfUc^SYe{GK zzJac6W)iTUc8$z#RJBUF|Iw_M9qkUh$q5Ucpc~kr4!k*_nNUVi+ir^I!ulk3Ni-`hzG0c(dn)t=L@NX|X)Xv1Z9qq+1By(UsS*$T%fy8CfaKG*P zMoQJ@w){=-(D%P*NS%lLJzDE#_%5-Q##dn(BF%d`u<>1YHiZsE!nLjbxe3<^M_88M zhU;DiUR=^KNqH-KXVyA>SVzh}vK)POE|MwiGi-C4*f zMv7=;M%+U+objw~$AvhOO~^VmW{Kyah;7sHLrO}Umi>V&ORGHDq@9CngEQy3`T?up z*IsR@Ugaw;Cx$*p=tUA#O(Tm5K>mU*Pj5s)dvhkg?b^JTtv-|5Y1j$$ zTvXBR+~3<8M$#PFQ_Oui${`{efl6ExjzopzAtjx%E*#L1O1o(Wq2#Hpvx~>F$Q{JB zl(T8uudC9z8`J4(-ztUan!IW-#c4vIfB>x`_Ln~UzaNTm1pm_ zwKCA%)_ZX*(s)|0#IPV83V|E$$w+f7@a;Z(pvKwp2LDn^n%}dlhr8w-1JpcRIpLv$dyvpX$)$e18w|qk^SKAg$GH#k>*HS++L+C48OkcIIncuZ&O&NNb z|6>04cx&8HJdyHuumd`qA~Jqu@u;3WUxm z=0QH5*JqNoCw4xk7kl^3H%4=)LCf@?4!bL<-L#zP&HT8IbEKRHp(j`MX;8+6o8Om_ z96as&324zn>TvB&0AA$6e8CE|-dpuq8n$f{z*rt(tKTZUqgMlv4 zWpT|n+f}Wu4byvZZQw%2@B(h}C0{oz4hQkW>zHID_H7b%GMs0ow zcqntGCz~sR^I5VeWtmwkQElMO&x;S##P>bL`H71jy>ID$O0`(1dSPW6cq2;xdL-t` zE@a<%@tSO0wQm^8USQ#5QlmCA+IXFZ3Xiic-D(ry=u9-$0FcJL(jj*fx!HA{?3*J8 z(>fX%@x68c->IBFG46eL;GHmJ3H09Ym|Aa5;|<>Ji>s#=88y;h=k>b{9qbtk=@SG< z%LmapjDQi_0!-dzk$K>za`D|ObL2l-fR)e2XcqCr30GCB#pq|LJ;?>^EcUZsqq{`{ z`cJP>>u`Br>00MGG94HKG@2t#JUD()wNaSEyl%!urm0YYxx+i@F#D06y50DF?<}XI zAZk$FZ%}MsVx3Xu4HH=g%w8tm-%Qd!uVY2j({Vb~6>op6EuJ>(=4RK~LtO0^P8L9iDBAs~-zW^GLd)>28EE zZ5wBYb4A9@1E`2!A;lE zDOTixB>1p?1IA=F+q|rD6R{ez#`kXyL|F&tXAp(7lVABx4-gG^IomPLl4HdJ{ku+N z3kg6OW3ZM0b=A$^<(BcS@NNgIY{yrTaF&!*u^AuGNrzM=WP?SB#iZFqOV#>*z<>mOV<7m*hEm0N}<(gq1i`iFn^50J`PMkM3h*FW6*nH zTY!=b71e`FgPp2qXw~|u_Rlz#4-sx=+8?fGsj?t- z1Kv1;Exm!AtD;+RM%y*EoPbtP4%m3$ZzNatC783D7Ei%0eT&_jJ? z$i4e)=bgm&&{tvNAQh|mBaaAhE8VCmP71oSkeJ=S==}%N!y}!1r(k(H{ zKnR<5RR#wVr*5aNLDK$ou@I!EnON^P;A4GuEt@6g=Cd~b!h7)E#IwjGF4cj-LS_D; zNpL)IucGF(-OTE85^mahc8)kA00eCFEUGi11E+We1XZNj3wseJxNerI;r24bxlfRo zs3FU<-Kp@qJ;XsGH_7%3B{dnpsAL&x9C?LBtcOXG%k+pIN!Q(hW0$0r+6g&*GpM_N z9-1LNvVGsI*?aH&H{j$dNHn}lhz8bv+=w(bd3w0C|GTT}**89MjUwrH0%guYI@HOH zkXShBf_?JXb?+C%&{f%GB6Z*;YCC7-H7C7<(vx-w+jb1K1OYA_Our856X$w%kCkqC zpSDYU?Lj(z4pJmePH4nX6i#n6u)PiIs`e|C4GpQhXw%XoNlPgl_=I=3(wAbt(J?EB zM1#y7k3-WG+}2ACyZ%j2jS*vsu}z_9?#|VFVo#Qvp`-hUt|@kQmb1U6r}3Gq8-4p( z8pElU^la+b!Z=k~cZ@~eMe^I^*Dx#&%+L@W&H0laBKFs#4)wmL-wsYOd2ncy>uNRS zvrPF%cDEw0QzcI>O2}LHE>aE-T$ZnJUBVjGN{SEsZ}$&9%4gr=pD}kIsSJ&6ZiQL+ z*oF*69u4Zq225b=!Lzj_ZzkZFUTMKReQUXi|AsUbJ(^15!dPDz)wi^}p@(u+m6 z1+W6<>=IYN{@YxHaB+d3YkX-v_~wB@=Y-T2U4e|80ez{=ZfCNhjuQsnP4D-? zYOiA{i_lAE>2;oE-RAW!k7XtK?tLL(p!zs~Qtav-ejw6Af82wF7u-ge<5Dv)w{D%* zSX5eXlL<|^0K$%LH**^GdhFBNwLHpE7jpCJhN3RC_VB-iTeYNg%>70fH!ifQ$VF5Tc9QupFds=+0F z??%QhyOeJo&o2!jDaI?@`sul-In~+tuMO0XXomL7=++Mtrg*ara-E+xJ~6 z>Ls?3?TJdf>0M_%1ZoWx@FxcjzIYyc)PRXK2)b>N@`94i#*&@t1(_p^mmZ3UzJYh|a~ zVY++%W7YoDpPwb&y&W|Vp0_K?MxY*4Jl>c1st(v6Lzj=b)ds zT8o;B`@v-Qr|hU~yC3!0fk96%R^1;?(Q|*v72$7&kWK7|zPU5)v`53xB>}SLPs8S~fx7CyLClCo0vzS}BDuL#+qi5{J)V&S*l3c2txL&l zAY}ITF1D%50!hPz1Udn;WOo~~wpfNB?eh1@YxpK-KfdT$yQuY~TUw8OuCbr!DRcHv zLbH^A5HQYaa5gIm9p-a?tGvP)Stva^=-!57<{};m;fs^)i$wWi*`yBwMaroUADEHE zpgdYX_;et*xzw{h=^}PIUr!qj$ACdpoSN;_R((;pVMw@)z@+Q}m(eIRSQM-EIW8Vf z28D$&t)$=Wv<%#Gqc)q@H1A#~CQZ~>Pi>tThVz8{B(Xn9K!WJr12&-(Zb{!MOd=kC zpM`Iya^yz|-4*5jPYJ6W?NccEDYx7 zcZwn#*VbOC--%ENL8-=|2Iyb!(K~~j)lMWSUaR8vL=RDak$n06#fwf6ZX|^c#GikH~1NpG-UBYvryd;z#7p_ zl%irbn+y<9FfM*a5;^TbXFJ6SJI!c2=8XG4V&X@TA~edrnLmhaCNf44C*3bS)18v$ z#W6Fpu%m?gTIvPW815tQA^W~V7?ncDZmSHM_0R!3<=uLLWVcAzgRpGnIw*h2GT5PZ z$zhb!t3DZaUA#d6hf{}ZB~C-U@(GPnarx{rCmn={Pu3j8p zbhmqqTEEt$GwVec2j7uq-F{t3T6xCClZb>4qje~;phrsCD~2<7l6BTt#5#_j!eyvNF|fwxfVN;D=5ErkyQCx*`=EKz_2dA zb-`;h-WM|l0KVnQcXg662vZ23T!GFXx6Z5+m-i3CCVSufn>JQVR$*fstleRkC{>+d zsNl~?`>J*K|g}giAU-U|d98kDzD}n%xzG%y+ z(gaO8LV&G~(g!v#90iPtp4YiDzucw!Zd#*#-q?CU#c$qv-}0E~0VI%Yt~k%CL;a0u z8)+{Ia|oG0O<%o~^`T~Qty$GG8=I2}eAu3XS>qD^x zlix?L`T%pWVF)+f2mkCsY35)usIB{xKEPQ0NH-|fb=17_ON;@$KudI^g*=a|NGr2l zZIr-i))?!_1U%p6xevhp4!?Qsc#Y>Y zLy{+U@`V)j#^4VB_Nz)4-09^(Kc&uP=GwB^`vWFCIz8Er`A@piRzH2BaG0$y9rtHO z;6wU=2TsqAPMFF%dwKbp@QB)go_w%=pSEF-PQd|g{Y zfL~9C3qikOs_g43JDG#IKTF{+^am|Vsmvt58NYeZa4QFEu!GJlPS_5=@_ZXyZMXCrI9~1;eK|DnNIU4`!&}DMgTky7L;0OP? zDk#4ugEZ(OY8^M_@GD2L`de)uOB@NIN2cFN-mtCJXz{lmz#%vC?hzdD5%EJQi%ILg z0jZz$&c$m&{+V)?WSa>}O5x1=vH%C+Uo%2yhE^KQ+8zIobMo_vA}K*0N|(poF*(YR zYVIRpuOdaxVUX;qZO)G!HzM(>GlQQEwK@L~O`JLSMXsq^$6jsiIm~|ap^74}fx!+I zckYhtu+p>us8QE9+Y&{k&Suc@k34&Pa88ra+%8NzAOIF5qWS1u9gYOk>UfZ^)}K7% zS-X7ifPc9roBpieohSSLhb;BOK}5BtWGVW4NzWHox%7izYL8wb=;bdK`-~dwaa*@B z-yaCj+<+9s<`7LS^E8iLLk?VeJX^iA;+Br^* z3_aIDcU*-4KTm6GJU5B0aB#uOP6yDR`F*UJ_H#OS@51#k|!J3OI z+NJAq-&~`wRtOx+^IS?qaGq)J-fk0cR=5W|hQUhBfW41pCr-4!m`eKk@TP?InS!*Z zAB(ZO&8cx}U88-Q6`XXzCC(6id=ct*WThJB9k_^8St$OO&>Dw-pfNR-nfu&t?| zhWZZ(MTJZzoYe@KT!;`^3l0w6ydAkivUN%a>~4brx~M?&6tOfUBLRLb>Bv6#i+5wx z$(SEHN6D?o6*P!D|C=}k%$r$qdXZl~Yy$!Kp!bnQe$X%nkWIG72)9@}Y4aB*NPh;_ z(8l${%wbqQ^k&QywhI#r5Fh;Yz9t+m3cHCsA6@LW{{;>eWfe32BdEX)@ImTw(ZQ*xPthn(4)Bo3x+g=6xZ651lCmf!!2C=ktTZzh zf-CXeeb52eHVeJoc8mm=qB?3?zz37EuH!CqqX^D3qxD{^{ao6cspv%X2WAzN^o zfa#MSX&PP3)+@>8TY}%^Q!afi!0}SW1r+o^ZWz9CJd7}Mr>FH1KL|L6s)nt705fj` zvmGnm?&P{U(aLC#uES8;f>`o&Zwc~9(lyEhTlahhBkO7}I-@<5RY#k}RXP4G2+9Zl zWrBET=kO;hlwGOw{95;(c}X$H;9<@E3k9JE{u>I4Wce>B$mpK&=m%$6#ACA$2s`9` zSB*OU$8oY!hv;G5^m>aAt7|NHt7|H=agG%h*U(`TDQw^eaM7&vl9yi4wW*1qAyT`t ztjGboBHv$oDVxWJtOqL6N<}B>4OU^R*>!gOMFMcVQM=zT| zE|n(f5vj8EV3_O(O29rqIV*G;P7f!SKbIeVEu-_tdKSi`yYkTrYH=|iqx%u6Kyq&O z$GTp#P?)N0v6@9a`BAV+Caro~L;Rg%V7meNA=v3sBmJVIkeHSriKBphHod4rt)M&BAtX!bhz_)g=*!nBI(S|6(a%kgK1``E%qYTC& zc`fD|@srYo%_uddBA~XwFW!(VXBioArP;}Z3}vK!*4o)a9R51BywMMOk&{eAPani^ z`tDge26WEg>`PUl<-ss!#PMR~y4)|~YV6L>qiNc*XYQXTL#6VJGm9#~9NocRz#LmGbCx%;}Mt9+_V);lB4SkgKQU9Qyph5FpN;kL5dvj0*R{lkMd1=~5ZG_jtog2If~8I#)1P5OkRA!*6)V zH91F@4?s{II%1xJZpL4oxa^3P99|dgfG8d8N__%nBImGkKMHZ{ecg9Bcr#WQ$~kt> z=a+%TML8-(afY$vyqly)w*BfDUMNl$B8XL89uJ*A8oK6_PWMZ7>viRS-W)WtE`KbM zN_X;dr2zl}E&OC!Pn);&l~gw)E|38 zH4AhLwQU|J$7xfMRUBZ()qObD#|^j zZgLt7x<%mS2KX-1fqIMpc zlgVMWR$apeo+40Rx?OvJx03A-yXL(-8W6i3W71&EZN6h_SY~Zuk?XBh zU)HIB@P{Ul3!)==c!tQzTe=HMVGq{qpgBIf+x3U2C`(+H+dbUHZZ+!gE!NnNC$%mN z46@&vwl(-z0ELP?37($NsVx7IIPRgun*xV9dylM++O;rXDsec3YN&Wz$94`eypP4f z)O~t;wX?LQkrs={?n}6P$bUSeO)PBB8g{`i-~hgRx8*jbihN;Jy1ieDl+1H+zsPq8 zVH5IHMqrh9KQyvoACg!ushOBZ@ix zA3J<3uLXuF2lRus4gMXB72ylWFfH72Gqh_OFgjMcYl52bUr)YufBgjZ5M~bp^@$$I z&b(A@_4LIWHWFganr0QY6zoMZ;|IKDKjNCVj|)$016KAWa0`#^V8+(6*b4hE1B3}7 z3H=!Bw3|&+LGK3@;wVTD;=4w{e7d`;WMJjh0q1Dx{Mgvjd%E=o>{C^~@m~lL;YwY# zq@DCMEH7TOr1^dorQtWl$@*T@d0Bsg(M2bjOXt=pefVe@`zjt`Dtcd35N#eJPRe2{ zQtk8)T?*6On_G$@gpe9J#&m8|24k|7MWzqI>R&sB3U!Yt8if(F- z_|t5THc~!))A)Z&%jP2gvLHfIffusAd%KbK7pS}J?RyhVp=UV@#EJeRqDP<7e4bQ2 z^`5EqkqNl|+V=Xw*|?Z1!@%+WMaJYitN_*L{GC!v^4|0#-xOkZUjx(GvrTb^9G`d8 zR3|Zc7=1Fd<=^u9tS4Ab1<}|n`V(Z&2hVKz!0AvIgdgEB|J-Vl;{q6nKxNG zObXR#!3vXr4J9h$Myuq)B!~G`$0E7ZnrS8(f&|VzPq|&PH7-7ATuDZ)SuCTP%J3$1!oj*Rd5C z)Q&YAdAZ`Gpt0rLqTSlRU@g)&a<`2M1y=fxg8~>SE?$2_v-j%0;@@;!N;0~fUcF$F zb)f*NPOuf+H#z@-GyfhGDRVsO3vbiP13MBz`1ga0zGvoMi0Ej@j}{#q+q+SYS^`qb zlBz$z<`9N&dnP9PB{eB7#2wSFa9aRsO{kq?rmw~s!-n&;nFt016VgyUpaz=bZO=rW z{N`-j`VAkri0Ezqn3qRYN6&h!?i~Ul zPEXUhF4sT(UnK+AW$U;7>f)OQwJ{}hx8W_bXjP*>j|33$EpPv$1u*@~)HHuqX7X@B zvIB|!z%|hoDprNKXgYqYtsNOsnHx6O1FR)D7ligF^jlcdC1&@ni^O#c9mvCrQ?&iY z`%Fmm!C+4Fe4TD><&5Bee+!8KuJz-Oj{nWpBzqCX@{s$$cto4BXfr^^>ou=}G_dH$ zHUj}JKx^qbL?cP_S zBSFoy<%wc#>*-EgNk*pPhVTDgxb+oEF~OK>|Z&+T{a}mnA-6_hnc+`EgUg>&&2xoK{DV(c}~hwaYc;WvL-YZX^#BO4Cs7t!Az7 zVQNl9NAHzWvaZpwDPKHh0EoWEa`w( z^1IJaU7L$f3pV;%+UE}%n1W9a>~T(bH_^5V;Pl#qI9sU=U|_uOJFyb3UgGuTVpe;x zlE?!WZn(tC4mW8KK;ID05?lTSHm*MdnK|`ZpzA7_g+xxvPfBJ)M>j`0fanA&*}N}E zt8LB+9mPJRKfh-Ed*~d_lwQmhUHmxxbR2NXI0GhAO%Fl`*2OXpXT92p9Zg!)&V7u8 z?Z0}Plic_SW*usxrbObe2V?_0f~t^)0_HuxPiHw1nCMK<$iAj`gaX6TPE1tNl zPdCDv0!g^HUP3;>p?zX}1~D2ft*%dze&%C_mr2So$%9?h#m6kA94 z9tI#@)^ZAXh(Rp_?SO9B5um4SB;RG9R!&M6_&0l$u)FH*U{v!szs`9s_-nyUf*`rW z?OE)sE5O5*8dW6fI<1IgS@#R$-3sG8C1-Xo_OTamwKHY1ul+URaM$wEUewx3<4V%( zA#EuI327z<`JjNwkh|jhaF_J?1HE46DgXA^UEm@VIapCG2@uu|9hiW%@VoX!$@-04 zn?okAxEeXntHir~omKGW;w>%?+;r@H8s{)MXEgo3qdf6cd0tES%6S5+=;LhQl^ga@ zrYtn7`V`()4C-3+u{`+Yg_I|#V)@5W0;*So`{;xF27@RTB==hobU^iX=X4ds^$dz* zv$mck@J_!EQK&%QVGu2eSHN6HE~|FCuc%UAWTDDE%+@-bF1yq0zy#adZpeBDpUm{V*Rxc|t%po>i6#EYdaz>B9U ztTJo3%(cprwLNkfy1A)hRa9yEpL`5y*#!@agMQgk))dfcNl(siR{!xmXx#F|yr2?j z1RaKn<0}QSMrEAZ=USacQRRqv(DKtW!QcX+63z=0;BZjEQ=rI%;@&+R20k)YSrR9* zXKM{^%Y2*Te{eP7qRa*qpWWrj6w<{fL~`9?)#)uOY~;epSzPNbx^*X`c@8ZY6xbzq z(3a;n6Z?N1^=FoXdHHR2Jq+NiDgx2!pU!&Zq?uyv=53_7Lj2}6AAXwUJ`1oJ$N}}m z28Y<1ws|t2aD(Oo3DgaF9b=y8sAWcqi8#zxlj`?sV?jV6#5mSF2d?SVqrzm25C+{u zZVkR6n+53dqn-I0I#J(>QF!Z>vH0lcm>T;Na9l=hwNGnob&@dtvGeie$bE(6Dw}~M zHFMzowbOID4K*}8o0C$MGQHk^m+&;szBbD%fTolGkBU0 zNB2FS6z#wi3*#lzzC368s`*;|1^)w|>~K_gd}hE;s>@m{+Inl55?qbz#Gd4 zxSdMNH9V&2Kpd0QWm4)DmBJ?P3m)A%ALcA^RpQ^SKj}rkIMnXW;E9o%85rzB=q9!v zbm<_c2bqwERNYFYl9hu0mnLanz77ZUv3GVIu%u#FxfANl(ljiTMRdcY8zy5BULEw* zp7gSRa_y2BjL`eiCXL=8Q9N86oLt@6FnL=x=LCy662cAubNW0g=^FC<0LWL92iA@i z0X?gkM~r`Cfa!of{)V8z_abVmsw93DRd3$#{zZ;Lv1}@zF5$Y-FJYrJz(_0B@=5Ho zKxx@2IfbU*6AhsY0Vo+N zj+t*NWrx7BAxxCQ%Zui6%@*N8SNf)fEx|A`}%bzZ!#p<9R~GMh5~MnV&BQU2-uf*C)kRJ)>Lu*c%LR zF~LOBze)^4%NNP7SF4~L#j%P&ahVVXK8@(kLz?>^cp%F^hKCbx2+(tRy>^9p@Z!Lel ztWN@$9uDcktbGm^;e0Ahr?1x_9Y<*&6?i0?fJ zVp45LjsAg6N;;y$u@(_|Yz~6@2RS zAP#|hLV=1O0&(?{@5vI5i#QL)Ki`>b*+n>L&`=hD@VoCn zl*+^Sg{>+cumk$^k@8%4Nw)5(L&F$cYX@su=k%TFR(wJ#F2y{(&`;v}uT~$f3y*X9 z-;ew$GdP7-x;!>C<$Lp7?{GVp& z3x)5OB+$hPkP?XtAZs0PrEIEgQWBq~-iO5uz>*+}N(%sGqnsnztCqym6Z~A~ZYm|v zc3?o2d?Ea6Fv3CqC9IDHJymIqnl_w`)V0~lvdu-4$_@1^AuzCVBl*Ln++eGr#z*}b zG7opol91d;znEU$IQ!0S!uCovAly^NX#EIdw^51YdYB^ECUL0J3ouYDmW#Xc&%<_v zPyP)ZAo`@G=}~C|CS)RzJDnA19xMDVzL#RPM}eik_k+S7sx1f zC);d`cZj@xCr`Jq7m@W}HApixTUL{Gjd_5=-9g7WIww*7>xtb)gXo7|qL-b23kZ zF_X+oupsni&c}>s8a}%c!^&|!&Pq?w&judqw)D`tB5wn~+?ri;pkqaAWuPUZe?4V$z1ey~TX*oC`1i;o zu|doEw(SuTE7{LO$=bYmy3iIo@r23{lc*-b`|Z{6Mg<60r17)q76B6V8KY*tXN%2y(0sidLIC{ zs~f(Rn?37KB;_*)#R#zneDHQ+aZU$hddfp4*|Y0swwbEfeAS{wCfZP{X`8OLqQ|YWo-ifyzI|$fUpqgXXK;cI^@^Q?ONODM<(}F0inJ|46ir&lp zEyZ%b$ZGz9O*_KGO&k1Zh{kg*>O7sT5+y}R&x_mCH;<51avNQ_d6&Hd&g zNzzh)cLniNOn@| zX)Jp%-d*@VTl}!u>!$*D%zlW$_f`RBch#1Dg`{HlF$H&r$&VLpmOZLVm(bDslm6ez zhDqWXg%&%SK(x=aHLX z*(%b{H6_0fKbASSejG|Ge|DY< zdVoZCYgHD!?M4M*So%_TRCY>W80aHH6;h;4xPV50Hln#QwCD;6tLR;Yz|-g$4%=YI z!0)K-3iOPHow>&tTSuXY$@qA^iZ$Z$bEbro)$eoIirc8p471#O&151zJOI7GA8=p) z1+mN{?C3Re+5PjvwToZxpMsqwPa1MtESeO6xwIN63epu||nr82^qe;L)v8(*lb9NH&-m`7?^+ye&Q>YcUfqL_0j2Q@ z*o#WdIb`h?&5EZ5{!py>HEbl(RraV!aCd*pd%U{XHjOsIo`ew}&0>?7xx%m(ZKTB_ zgqt;AG@jWtoWm+&t+LO)=Ya@V!?QtVnjUo%5+zPsn>i59V}dQsIUPmne`Ldt1WWf_ znCAQlq>k6`K*p4wJkj)dYD*$Ty<5UGgi!WbrN2nTN)>y%X|wt;gYN5i(Orpg8=spM zBo{r@&TT!*d))rM)0KB`yBlrRBM9qg7Lgyk3YvLDj~Y6Ql~F8M4Rxn;=|4#VaQq=r za~tD2OlFB9zr*V`=4Aa;jRF(05O`hl^1vH$kqM;iKdg+$r!;@CI;@&$`p2+5hlc)Imc-lM* z0oE&a-p$&Mtcxcxnuh_!r4A-FlqRDt<^GTc^?&K^D8(41_NU=MB85mZd$>lI&rB_S z3*1gm&?&<5DZF`@I>S`E!|r!Zlyr(DU(M*WVG);L)d{=}gL}3!#c^ez4*J{119`tC z_G5R~$*9JG=FPKQs^0*$@}sBupEjP03){P`A8WtNQ_ZQR9f$`1x-i zkD}Kl2~6aZ>E^$EJcm_(`*;k_>OJ9_Sa!A5MgvyotAVW6Yh^U&W&3qS%?9U?tDi1I zduN`8K%rvitUwDkU-$dI@?W3!cK$~l&t0o7dFSa(U&8s?wLj0)k zQM&g)7iTUrx9{P82q8sbWqI2oUmB8+tBpalQhSQ_E)6#N2^I+ z(%l5^jH;jAG?RZxPAxI}&C>??U@KK21V9(6@*zTGBmM0bwU^5rwNGuG!9Apq)v(yDFzmzR50fu zKshVtJn<1c-{{!Hs3`n+rVlj6s6Dib5s!xOCy@S;HSK#8{hd-A9c;TMLL_IKTiy{m zXhLQk!~3d?w;p5(M`El#cf_oXr!NnBH!;aqfkOz)i z%?;;ay;k)HWm)w_KElYjPhhRn_YK@ z*qm9;7bAGn_A(zB)`G_uS+5jWL(Rah58I|!qNh5>!p+N=p0z~a$;UI8Ri4$-S#wlzUu$C*{0X_e=#f%tLRGeEXn5M;~z7 zVwleSYx@OVtGXE9GQ3d#5D26^1}K)eLQn$QOPu z2~+2yJAr!EQgeNm`wN{mWTNT=-q$jwtHDP$`D^i+;=+!58B+8zekApx-7J0ohpw-V z%W7NxmhP5L0qJg}8w3OdlF)0C?(Xh}U2` zvu4(wHS?Vrl%^Yb5GVtzy^Ns~BLcEhUYsoYAgT>yWdNm;J64ZeTR(+6AQClhA zTV90;;z5bi&N-<)<4KLAb`@8RZ&tPIjH<3|5Y0vh;CdCw6Rl@f_K~Nv4Y#2I#og9k`wMi&W#r`+}>}|u7)jLKnSGdR_ipwmSiqXs0wI0{e$F`UE z9Yl;?_}zR!x>qqVQ2%xV0|TqMG|lxuRnf_n_smv3rYec5u|4477sLgk-m96%6CF@F zvPC!`0nOvxI8fUQ$dnibP%5SIX&2T_pPxZL->Z1uI%_WOwVLEjP083+h))Rib76(e za=i4wvsFRnG~K_qq*Th)6k44@p`>UJyFSs{GoKyFltxHrX?8P5!8F8^t_2DOK?QGN z%c#C9DeYL_BRcQ59!)!NeJ%==jY7C+!sz~Uwm7&vpil5kW0mIch;{SozLyMcts&I&6U9vDh(qtbe$E~J0f0t zi0({h>uPeugX19*wEqf{H-1SZH$~^dR85YsnV0ZllyFR`Dl=3q?d0d>&l#Z-YV$4& z{S8$)6WjkENR|CCLbMiv!U@Wds#07zJy#V9%y(K6@zsoN3!4>lTdI@`&O<5u2y(S} zwN?-%(iWivSad`AMik8a*WjKKpMx?^hnP89m^|-GMN288{es9Yl|JOE1W3U)ENcN} zvY~H;v)t0$F1BWs3u|l)v(Wm=kP2gqC2)&mc3PN`lEJf7puNP=`bv3YS~XlJ#pJ@H zxuttX12_oh{8x~UKEtD89vy>7ld(8GG^*Z*SU+|E4kL}d16dnn%DGf2Xd3O)RfEk? zp1VFSnploU-WZ%Cl#+&d`{7N8Lu{Qc=QIpTSpbincCs&Ea~!&L(RB9MQQq_9T5>Rb^zWv{mq)YAAKKs5jsz*VX6D2 z1IV~#(S8kZ1Rj6uqZl0QV-DY%mJ?o;{C4;V60Mr*tGBFY521hdvOt%Pd`eX~m{@RX&D)IOR z{XFtS0~)OteI-cT@7Q3tl&+XC4QfGpiLY9TH z3?BoCoLw_y?nfoDqu>2iuBDYA4EY>9Bq*6nL5I>g~aA7~p<6gSB!z!x3ds}TtE|=75tQizd-Z|9t zM85d){1~>1L^c!)_S?`GFj*cppDII=kxYmv6LKVXpd3{IrA-bWdI!}WA8E58UEJPB zai0$tM-&#!no6))E;+wbZ8tM_3hHu&sp zVkj^*z0>VaOTWW%&xXR^jS&M8!Fzwg`aahhX$6yr`5$+9fOF0L_5}pKgEdjd^oDt6 z77=-^TID|wl|KV}B+Ez%7K{}KK=cYOpmLkvr;|S&15w>khvR9$nel;S)b{<8iN!ze zgg>`7!BV9Iqw8GWZ-`NJ;t)Xp7q#1?)0|du~m)mb3x(z6*A57z#ZVLqr`6Sb*imcS*9YS7coQ@%U z)r;4gb}_*%`^WwAH@`-pz%0H4oM~r*1ef*HQ{7zdN-sECto4(-*YPxv1e-axZ0z3FiXcVBpcE65G{Jh<9 z*V(I4aK8aR3B!Vh^kvZJI{Xgy%5Jor$~d2I&Ylr+`>*1Te~WP@82GH{FNW(Of{%;j zMcmH4^1RUS>~oED8)@}Itqm1w?9@>=CI!zfExEK7@Ffgz5a|dk9{mp?Iz$2+DEO;1 zlptiT=X;cF{@%U7?t&$?68?zYKJ4iP6J0?a5qes|$M zgZFLK#_acu>b1l(PyfKDMMwXa3w!V@X91N4!r(|`df0P(liDMG9 z;nLs0cr@^FGAV?raI}ZhdA|oxj=cMIow>JWHouuGu_*Q$*c@R2BJ8RCdeOt1i_2vmnH1A;7`19Xg?%k8W2&gV~gO-z!aK zmvchM`h0#*c`__d@Dk=X>8|(CG`rP%)jxj`&qcgugcv6?Wm`51iuxcf3#}fJttI_1 zDJquSWMV8xQu>u-$QX2Vi^fa+&vw+TAMEA$r(0_v18nuhl*#D?l$=IZHkfy6x4N;? zYZWjA#?*y?fsn?IY)J-V9o#?e7=VS{Xv5Hdc*;x!w)>Nz!6AL;X9bDXfi7{PVEJ80 zB{U>yYSm>RJXXFOZVo;yXQqzIzxE{}EO51me|A0JfEO1^02LyA-dxtwOj?46@R3m(Y-o zpIu3?!OvihT;SmfyV>J%GjDc(j;rH_556Qrge7=^ zEWY?Aiw+sFh+Ce337Mei&*QzR0=+o?H5olr@Nt2>fcM0PRdjt{jDfVqyrR=LwThw4 zB-0Qr-e1vL;EN5cAdv}RBI`75nQEN<27Vg<-Y}OJQHSUM*&&X($OIpvT_`SOcT>j& zG(jo|CS0(Ee5JP52WV#+HX~0sYrxph@z6e-)m(~aFnikoVhI5gll;+Ihr5{^xpGvG z3Od-k*Q>W#pBm!Me%@hT8CvPjb)NvX8_7{~-n)`%{C?@%qNyu^b&*e^0hi8_B=e|O zb9j`AB*~04zPUd57*{hn9%d|7$S^H(un#~`&^`vt;>agzCZcH-~zF+2ojD`?8Y(3`6O1zwB$*+`V-?r zd^Ya#F^fo)$W#zA9q;;^5~?$r`0H{agBX8hVf(1X&#=-p z>2+LXiQ2*FKM$d@O zFGz_TAJGEwU`=TMQJ=r>S+)@Y%$bO}Kr>HFFw%@a$E80H2^J(r18VM(gDEy2&pmN7(;iiN`ON;M*OP z{EzPgUqCf|UK`VHwze#B!M*?4(;0OAOk+j=v3e@^;eXx{=>N z8)JNA6W+w57-@`uzC;2|vfP01N$@y4IGrfKGv2@IhJv_gJQ3zZfP;3|XEHgk+x2LCh|o3cKHI1msP}m2i5ijz*OA z7r#*-P=~Re_b<>h*_WvUPvyA=W1ZA;p84N!o(-d4#!)8=NmF>v6Vtb*pj)ly*8-}$ zfne9$u3fNhuL8>6h&6o=5FHZi%NLEP7LVU==4BPr8^vlJXblQwTVb z?{ip)wjRxv05s0?3nZhO_E13q+MF-Sg)GSBn{>Ll!j4TDB`*gwBpwpiFo8Av3#hZn z0`)_%h;(uc^S4QPbpJ_kciiGzliw}>uW_m3%6r5m;ZQ?VrkUU)q#mjGk)9Wb0+pW>Y`RQf+nfTw+f5(#H zuI&5~_n6|D>|XdKq>lyA5Y@ZfP%T7GzxLIuPCQf-uIY2RK0iG!*cxSTai9K9on_L5 z5)zN-(96WV|9*D_F2b7Z0ZR>}sp#P2ra(h|AV5by1w(>M{$BOSl7ok?_YKTj8q`c&xtqG*Qjk2gl>O=z1sibb1ft@(%u3H@z_Tb z=^!*{NMz^&9^Yu6_6>MM+CO^!YXZC!uqK?pq&`p&b~sY-A$B0M@ohug%C^kNqF1xf zvW_(>7Z(A!f1FEM^l zDY8s{>ViH7gmqg*&%T`Kk`8nfs8`KzBmqgiu9t+ZKoJ7H4bNojo6`v-l18Rb zl;hR*9#)l7H$un#*$8}2OaHu{8G#{3mau+v+Od5g^%$p%&0nhiQk(;2C~c>IE5zI* z&4Q@1r3`V4{U&8Cah|gG*>nToM`$ju1RoY0V>RTSuzl#Kr`~qDvyemMhCqWn8X@ zr#@7V-f=ab9z4uvJ=VSBil78&Aj_XypFf-1FhMUaly?jAeO%}+RQ)xErpW70f?&^Vo& z--5>jHonU?MJYoGftuB>(FGQc{kr}hs8w&gb;WzcW8H3C%h<0<$=sur`wBkZ(tB2} zA#@)KKhtestHj|>i=u9)wL==Tm6g_wqXAQa=hT*RLpyJp24;0|wO*63=PX^Nzuawj z^o0wy0#_jNzLsM7d`pY});->ZzHKBZ zGaGE>!}b&wEdPQ~J($PdB)yaU9G1he`QEK$6fbq-(G51u#7?MmLQ%Exnz`TjppG_A z@C%@7Fba`Xna*^s=^+zD159XZwUnJgrP3?6yP3Z2(XVjfgaUs4^jdB47OwLSQB`7N zcqU`@;mNZVKf1Pgwg;{}JG-O{R4dqKt1L*|LhsGimC6r~`F$)iIlFS`>(;+mXw{n+_=aG@#k-l4 z?dciZu!zmu2npHXyk2MAEZcmfO)Mq~-a zumNVI)Ey;?7Ic3$;ek@=T3c^@g2$jy?;>Y--CJ)zVR_Htme(J9yXvfT@-d7>X3d-4 zuE^=f=hmB1fghXm$_*%=_6Jj&z{52Q&GU{&%Qh5hWZr4i+as}+&MS)$kI8H?YP9%C zUp*~;VbbSYh3*g!z{(!^p}>4x(|Ej;8iCY{0zb8lm{Feaf$#*5DNEyBiTzpNriay`lQgBt8cwOudBJMg z<#aMqmwJ4zIAL=<%q&%g;&bZ}`U-;IVJyRgcuq=p zuW=Yr|zXw>c+{ zTNKUDG2a>WHfr6c>(^izv}+h_=ypao%y8aVzJ8pt3hy6&Vs(naR~)>TO>01F`l2cq zKW5dfM}W|E@YKA*S}gMw42<`SAeX1oW`LZ^>kHdT*VFtkmc0SG6eMEa+sgcscZ0QQ zDC}+o9IU10V@?l7SzE*i#V&0gMi{fj7scmc2Zm|%JQ1tcJqU?!Tw}gn5!Or2H+vG+ z;_zIiL~`kggeC?UhNm|UcKmS@MYa)l<{!ZulTwM^Y3k=34oyp&yQBsd)Cm72LjXy4nH?V zTCatu6u4MdyFz@zprxgbxvclaWp7ag&eLh#w^UM_td*;=isoORifWiAsg;gy^oT)Q zxvN**M0V5ky&kGI-4n&K^iwl(<}=-drqDge@nPBA$LQK09w|c~!gUVkM0$13HdC@y z-9M`6&s}J~XKp74TNVMa;p=`9MK{venPTzHtyF7~)!=es(xNWBhp)pm>>%^rIzO=s zKb0MtC|oL-5}ic9)<80qMWI^!4FhrJtB8$9=1(;s2wCPU=01v?u}zD3*VHwUnq^im zQzzQJ{vlfxq7vgK6_h@E^fk8$*+x}_s$gE5dM@PegBqVXHG1#|q>}%x!Jh$p9VQqo zV~8doMX|DUFLzdh9!L>jVknq#;#El09-nbWfSXDABJ4$fOgohuZwwZ_rcaK=lI>V2 z<{mDgKQxj9b%D0Wc{EjN%Gb^O@&Tnx6@EC|8}I~{^Fsl6q&=nfZXxZjpR&=Tjfk~e z9`NlwHl#Tp_$=d7*ZIc~^!A63iuE3K&PIu`yW~=!_sB(V?ejqJA~E0T+qX~3+S+HZ zD+7eKE@e?stzEkXIH1FoJ9AHG?ii=@ZHZWI!s{#?d3H(wkZS;TnpLm^er49tQI?4Q zA#Z_d)kU~%VpmU(A@qENZd*?3YD-Hw4Q5DNsYWjipTOg_8EF=?+cLIo&Q+UVNCNr( zQ44XuT-qC2cfF(wvyaPCfr;!)`aKBW&Fx(dsl~nf7(V1LlKbQ0<&v|UTQ|)Pt>$(Y zsnnrhzTFU9_~9J(-nESi4L_31?(H|(I2;>2m`t^sd4mohv*G(^trC0WSlTGUy($9e zIA@CYB{}FlNjokRr&8QXFNK9IT@NCMa*Qs5Ger7EL8MY{qURBXr{=~%q)7_-g5u`$ zj?)LYEe?y{hZeQ=WILkYG8=2<<;zo-(;+y}Y>5r#Q+HUnorP!V;Kel*S*T5=kZ;7)u z-XT3zY_uj6%c-xHEyJL|l^5zf*|kTvlm4PB+_uFG44DAjjRXm%8 zTfWJ~gZZ6odXwt)_CPa#3cGv0;_D)3iViU2KC5|Hu(gJZur;wPtSfDuS4yqi9P3p@ zdAwFj?gV3@r_#7a4MH}0D@7%=hVw;QO<=v$4w+Fjsy4Jzp{5|y z4)E#X6Q~qYrgB2l;5kuYq?uk;6frZJt|uNyorntanyz!#SK68nrqqRM-o4ALY@zZ@ zD_=bjGT(Pe+2VZj5v-4rC_kMgwynD%rVDuq5e{r4$v>RN(C(F`Je~QX$9Th1mw+9o z4>bO&LKiQ-iFrnp_s!9V$u?L2VvXe1pRO~ze9Inia#;P4y-0H`b}d*tRTv-Z@+Dj+ zXaAgi140CW^Vm0Gpd01O)OZ$1$)8z0uRi2wkZ{B;!tmZno7#1IBTEsX3ZRkg^XAAr)h$r{augrK9|OkmjNNH@e?OHC^kio*m|p(vdmRL*Su{8%*8 zW1B-MR7tnf;ttEpN9Wr@5SW#^#Q$tpew%;HI!!IJ)_+MgLGa)~Oh!c7^mw0^tlN7o zw#uha_Q~`m0Fz^5V6thAz2Uu)ebn)|?dNuKzB#Mq#oG_041t!mKJxf@$EZ~s`t-me zx(doz@#_0vlAA04Gi#1#!=obILH=YbDHs%w*fVmOx0-8!2q_iQE;8DF0yItYGEa&Q z5ti^P`cyNY*{&Eh6hjuLkP*RLC;ufjs3V)tYLG4WBYhJy={#f{(;;>9z=$N8g*d2% zYHPi02S^e44*o^JNll3=i7-3Z2&^vwo{UR=gb78i+*}2*B}bN>W?iTiQfOIiZ;#V5 z_^Q1I;iHVe+RDzY*hsNBSbH$07ous*+a5NqV>O0;-4JG84OVgQ@1ZSUB_9!~z6g*7 zf?*2A(0TU6yk>GAqBQM*H)@Bu;;*DscZ#@`8#vXI{<#1~nZ3-}| zd@L9Y5UK7$N9FeSZI+YtMQSp5i$~bvhwI23^95rWzI2>l{4oD4p@kl27LIvBgOftk z6Iv1ookZ4UwS)lxG?Q{(M{<6|V`I85o7blt3i;}-D_29_sD$o>1JMGQ+x>ht(xDn` z>++^XsW*1t+dgmbpfXyT^D5V&+8WCGZ6XDCgNj(8iQ=U*NBf!OQm6{MX^=3$z_5Pc zw#kpx`bAL*X_PLo-{76u=fy}S$54a^u;roOO-xec5VweHefofiMKEQ-zl~{CnGUo? z45dVfuZw5+_AV+=;kX{9>W~Y15r!V`CDcaJ6$e3Wf3i^M%(EhN~ zE(JhO7rI(D0g_s_*hLslD=nZtf^1YCot%f_iQbKTj=?MI$&|8N&&OM-#Cm(VCI`#c z=0CJ*3=+@L^o_dNuTZY<>y}C;`|DZtnlo%NNPDbHzXf&7Esu_LsBk@Ss`f_GY+x)$ z`$RO|#?e{6X>`RJBvf0@8CpDAeRqb~pIC-CUmXebx11rK-V%|+Y7fDNybS82cu(#` zt;B&5_qH;5N+LXM9nPG#Z+_n-`?oJ3^Zj6;*;L`#dXAHp>*NAQU26ZD5tBE_#%Fl% zu1Xdbbdh?rvF}W!T;EK%G5+HN1)_QtP@QSd(`x0R>PLRh1--> zC8{xP-b>chSQ;_mG~azDXXK4jMcI=&n8Octql0@e4Yh|v$9)(f9-z#UbQfa-pC`7-)VT!pWRts=sq&8#e zFR$1Ji^BN`ip4>LtxB#&Lt(v(fJ>8NLp_nnM1LEB#z22lxI{qrWL4x0T|3)mpR9rv zq{SS1m%f8tR3rL%>fNbR|AQ%Q0_n6NC2MB`a6)avU*te(aIHo}YhrhNxSEHX(b2Hb*7= zn2#Hir7-XraZw2GPhf{P90m?m28OYmKrBZMn^tv9Gg^B#bU8!Zlop(o9u9a#)^+ZaHk6 z%AJz4ZrbM!rLp_O#<9AJ2VLr+e%oWI2pM*|%CjU6EB0u*l=^(xhCk z(pTYnVknxW4|)*<&e(sb3a+{no^HI|+kEYb&dMXBMdQO8GsqQB2SCn(k=LZU!c z{PSeg;{6T@Y3usdJ8kVvzoW+v_Cq$!Wd8s`$Wv=pST^PR8EaX!pW=94&SR$eA`|k# z4C<)Izn+TUc+=;%CMW{C=TYM>HjJ^ln*B48#W>5S4Kx1(1Lt;wke*DG8}9s6W##~A z)CaH59Y(ItmiRJ3Uq{iKcgH{u@zBagKGVS zu(CnV@~MjOwm@I0u!yeQ$_2f4AsY)u;{Xn-$$f7sALe#vqy7-wIuq+AU62<1IjDGU z&~8l)N+Hvlf&xLccwNQ^DnWJ_bu`ZbHy1;ZHBSu27&9!|5K{zh z)qdaE5b{9pTTa?IlVQZylgJdx6bb{EOs9a0(NSf%UvITJp0M&vdqvl&P<7YnFey8P z%CJYm7)11z&739wUtVYiNb`qFtm8adiciE@eqi2h8&vXw;j2U&c(YD5g<$$WHG)+gUwMEjCOzh{GJkW`(m;8nuq%}rtYN?kgP z{?;J``cT`CCyl+%BjY<&22SL4EvWgI;jE=eqm&}u1?u9|FkG(vs;adIiYIFqpAC6z zPFC5E%K(fX)41#xLmRBb;H90igksQ!Ws>T;LiX0I3NZ(+{nUg9jTA|b*G zMvI=bFC??(q&OSQ*r!%G$54harlYPuBFLzheyU3KT~5i96|nS9XZ!ZK+H?*LF})Zs zML8QAobyUw;|=TGAg-0sI-RGE;0qrv!rdSA*qf28;|O}Y^RgM4Vj`Vs50p_n>NGKU zk4yF0JjYkDPwj>&u5{MaPzm79o1~8|-0ywb%R_f!-cC0frN~~{?PC>9zwQL9tvO-~ z2K`uvGt38v05_NihtNp0eu4`Zsac(|-sxqx2{9%L=b6LAu})qJjKay>9*G?9|8g|z zgzk!*!lExccuZa2XxJAkJnPsSigGo->_Ny@JzD3@a6rfHy7@AL*L=p~C1+uP&S2(2 z=@3{~>drfOMA&BEoxPSYVE;gg-k<~niw}*9iO{V*wWBLdRJE!>Cj8mI(}pLWh_#rg zo^yYOT?otnSRQvLK0MqqAj_!P2z{CC00}+cnd~Y639XiBv>tQEYTbms$jJto&DvS| zLeW^qD`csZZrIh=P(CdP^Gb%qKQsgr=Y+N{po-JXb)!#QGv@+}+C{@>yUKE35N zZoMMrg0DTZQw8D_5YS<5Qv!(5nm08HuNE%p`;@U+ZLm^Vci)w2o+Z6ob#c2hLjYn0 zY|y^kmZ$SuUiT{sfE}|=iwt$P#nwCidU(5~^ri!RFlEk}zHdU@BfE8m!@mYBd!Vb# zum)6SD8-lz!!k(GXPbTU#W|JD?$B1euBX)UEJrF;W7QOtf%f$YNT&MI2N_r+JCLlD z06H`y0qbf}NTFhIj21|YNzl$t&3FjaGUUOMHhpq$o)F>OYG;AxC9PR4>em_f z02AI_is{1XcD_-B!|dX}ZzHtXU7mj57>Hlk5l$G`lJyi<#FW3}hBLp;_(8`@;r;vG z+J`iVEiy}a^YNcwh*LE8c?{6}!?gVX%fmXXMw4So4&~OBdwbA*WmTDtg|%_LWbEj& zpZ%OJ!g8A~`Jke271}DLC6(7TYdo3UhF7Xo$2%C;Sm$ZZ`h~y)hn&XY&Zz80f-#MI zjFH6)&B>kUG&1S8GfyNj%$dbc^Tz{P&@IAwkY{CnDJSctupX(eJnp$l86n1`(n|GI zv9b)UVDSj94Pge+wQJo7=K~eck4~)xy28atJ%W71F$Z7pbWP0gy5d4KK?-UmQ}nAG z8N@JkHG;m%l*)B-x*O^CkF%FXZM@EUVFw}2;@8?nlCpAq+i5%YO~&Awe%PJw16@KX zJ`O60Hj$35a^Dv8!aOy$zyrTrV`EFDe3Vgbq?w6m_>M!dN1yJ^svux@Hg8ZEhDC{9 zAD#-Tp0ICnhvh5u5L+ISdCt~eHJjc;2&$LgIeC(v4rfeWQuVvg=VUMF#JjYe!IM%tX%Ozw z&n24&kbmNP7056C&o#8y+FYxav_o{02qtK&c?DCdznDDy!E_OsL0azBRPhJz3W_2? ze0E+!BQZ@q!5#)m8nM5ey$;OnhL*a)J?5&!C+H%jj2a48A5QM=p(6SGXywQT7OA(I_qU z=ua7E7TaGEzo!giucr{hvfF&6{8IybZYs2`QC6|owgr`7mO~biiR|$j{lUZ5naaBU zd&H5SkNQ(fAQ<%q~DmILDkG&Jqq9isV4DVyPrw8CDc#l#5si=Tuv zD;j9h6P0$~Dl32TH} zOWi|Z*g*w+@u_pMm`+YPQMr%;clJ^{C|$u9)?=bMC2o^3R6?kXekOFFY3XkTEas6v0_9U1vR!!&dy)bM;3`SIA_zJRXaWn8_g zq0-hO9;r;wv15eG>ZgS7|G+qHM4k~Dk%ovlYYLiA7fYx#BILoY`KsJ|gBvjOjLRj^tz~Bl>tF6b$7Yl$Ke*Yjtfe3m z-dJ-hG|m^w8WStItNp=%0ohX;ko$0FxrI;S&Jdp)kjYu@hJ$CK5-u|fle`5W_(qY(@%y zVXcl)QHN>%3yJ{`Ub$$t*P@WEIp`>A3gAk=;(hpw8OMTxjK5vHe2LwJRSxocD zQ%Ig#bGW$%R+}mRQV79U{|#cy!~j>A^+HYc8m|N%-mD+s<*4!-0rmI2)mE_67+aU3 zx9+sp{8-gnLep;m&=v1o09_^eU-SUTG7b1A zK~S!>nISML9kPr+lJt+~>=`He$|WqJ@&j5R4Xg>zfARzP1VHwl@tZSVW=@!3*x)Wo ze`MotE&w?nwy^D75UrbHOTtpjfhQ~(wuM(U0GRhbFTwIi@9w;9g=C&iTvS*m>w!o4 z_lTLLh$eGN{K|j8!@ps~dX0$k0%1}w$p8qLsaH9p0y>yQ?Em2b6?g>U^Az{}HFyvK zO!9xi6aV@YB?gQ?N=)X4m=Amw{5MAK4+oxwJRtlTP5EYB?d$(|0zedinE|-DbNn)i z1-%08|HI~AToQfe(n5w=z<&wJZ&~SnMzD74Uz*sN9Q@C51~-!l0yI!??F-)huSWoV zrox{W%Qh?LUs&!x;_wU{)emacHxdNVb*b?Doptf=e?PNho)vtg&&zsfVFDZw8b3iv zNm$E;T8AZkhmBrkjYhYP#svH%aJh7TpQlIlfGmj!F^W0&Ek4&vfi3S5MN7U5byuGsiuc@5x1+ zwHn~V%LM9Cz65|uP*7^qa@8JtV!oq}-N~ueB^dWU5Yz!(GXvD;2a-97r}^$Px^9QR zl5u(5i&j5`$dG!PRaq_|^D~mh90{Gw4joH4H=)L`8V~a4GVmAM9&GJ*gd{XCMZ$cD z`qdL%3k{SD4Ov7#ED4CO`10J5i{5te$O z=Vw1@!aNjy$&E6_t&hi|!E{u&22kIm5)d4AIoej*B#7KLh}e%i@?8eRGc_L1?X*?A zrAqWVK5iCIyK{!D%MqXP@*jjE#KR*KjizT|%6P<4rGud42>8*9r z2j(_-A`$vEK6o@~EBjL$Zx<_!ZnbZESmUK0qdqP4v2J*9#`HR*R8wb9&$r0w^+a!= zI}$Cr?LnOH+$FQ_h94h61bB6N@t~6ig5>oFJPd|ZaducYg?_Mc%h$Z+J0=Rw;9jBd zyqUP_j(tshx=tS+n^(40bIB;43yP;whVl!JMXJ)!4q|KeIN5kA_Iwg5qSNnu*N!4^ z*%?ni7s>7_RKyz<^bKu!>d{k_A_ONes0-h!(Nlt-U*dXBQa>s6dQHy#c;Ak3Isq+B z+xddf?O`B-|Awn`ze&#D8hw9@4oSE`%?UO@@i|#AFc1w#v5GDRdla~Sw0FOewUS!tOyycpkEf7L^H0!r2I^bA zZVw=?50evcMwzQT5cCnJCo^n^uL28i2M!#osa17ZpC*FQdCaeu-aADQS#?(k1Pmls z>0PGWWKe9- zL-`t|%Y6R22fY%d(V+8P)7)8b+)rlEaiDdXz?Hd~+hfO5KP#;*YnPIL8~rpnGA_#~ z0hk_>-J}Mbz<%2Tzao)6k3WiZ45VP&zp{b z?Xf%8e6wl51|Q9h}e(KBlgL#Qc~oX}4*C~-9L5Mky^Ps1@i{6UKV@^Jnt95hfiUqSc#c+T?zr^<ee_Yau#YLaO%tVV-yWA7TCUjujkUQ?gPy_PYt z!jvG15W(+D_7{vv@c1qeX9b?%D=n5b3UlR_yiG>;xxc10Kug{~H6Dclv75C5C;pba zF=HnEP2O^gE7ILbt02y$Tbu+2Z7lvfCgVe4I<@-we)PeYT0ChNmj|4Zr|ecUXl(%z zao7pib&YEZegyXa0#BtSjUF{yUCfF>R%$}>7OqAlz3QR$Jdwave z!U~>!PZy5s#bmljPm1;P#>sqLnte?k$bv^11;}e92=tbCT(_9^p(F;)BWUyCdArCuS2WuB<-LAgRBv zFB4wlY8Nh4+5ljlSL6_INBp+sAU2z-;hxB6kL-o;Y#pZn=xyZw+_b%}p{0q_ z-MXo@1N~(N$W*RYO=nTvG+ij0>72ailmU1*(nvA)AiR&^b%zW6( z%#}vWh)%7-St~7dF)fO2>C7X?7!nEj{$cruq5gjDh_j)#d+qfMq4G~hjA0UC#I#SN z(3SmRM65z-gl>m!r#n-U=C@s2{OS1C_hkO}?oW>aTIhZDRzup_Lo?xob1Mnj9&z<9 z&jz|2BvHRZvk&zL>1U|86>PrBNjg==QNtNcn#ELW|GVmW{Hc7^2-C5x<*p+EktY5n zT!?16EJ^b=X+}rE6t0p1^J9F;SSGN>J8i}@;R^zB>WBf)k zJm{4Ql_pawO~*c*fpa0GyWbWnaMkLpFRhhKQl0IL<}foJ&>F~YH|1Ec8V|(q+HEli z=g2Tr-7EubliD$Q$oVtd22<<&vOOARpPPs?p*ZY~FT*8sSVfhc9pB;~g{*d~epT>{ zfY4s#LU34V6;sOBfPq0F?Ce{3<6~+u>xh2jjVW$=Xc&rEW48rqTym2+;0~Vxln@f~ zfpXrz77apQxmt4ZW^_xt2T6uc2rTC!GaUANwl zbHFPtc|hPIO%Cz)#rK07k8Jj*MgW#uYiE#*>*YR#_HphD9!d_&d@4VI}CnOT!gtqajsAVH7qRF5%`5Ij=Vbezcv2{qm z-RH@D=`1{(D*G*_#1Vf}51$xsw_>3=wY6jX+l}2g1i_n)>ujl*S>Av}bTT~Scicvc z)_M@9TPU5e)ML1aBKd8K6bE}3_#8oQjB4d{GDLTJ{nK#hZcSaupE1Ii>y61r*HAIj zsn0d4%@G^j#LG=a+Hw2@z{B)qzf^W?r8h(n&ONe1w}n98eMOg(`wsXqee@$K9B58< z^*E_{#DZV8$4X@hia@MMnoSmV0SAP2)n4Z8{l7- zup1s3BApj*k@UXtcs}w5zht1gI&7CxR46nzrO0v*NN8uej$*d`P^2|;4_y@v!YlGe zzMH$}XoRFWI_F<=rw2-GYV#^Z=NW8ury;o!oz_&DO12{){V%R8O?2dffi@k_9D{Cj#5 zjH8v>i!VWx?LUj^Nj8|=GrGiho@lzEE~ z@`YQ{y`4Pc*?T8p(B**u39C5Jh?fKf(}`vRF+f~@uwJY)LmpXdAjG%_y+i)HBm4c? zka;Y)w0qz1O<-b*|H7~+$X{HW09CXKs-kzmL&-gZB6aEL(78jJg* zw2hOkacYddC%U|n7`|Op6jV!xm)`mQ+t&qqdvB+lQ9ASuE(=To`p=Tt-TQ8t-!xQq<{B*|BO?N-(r7I&`rz+W!Z6jUB zKRLel8KE}l{5Gbq;gTWs8T8SyR;l3Tsyy*Qzj4O9tP(QIIMd%w!I=y=NBj+sQL~*lu%me!0t1|3&uB#Y#o=lfB|WczJ$z@n zArLFfDtc7~aFU{FJ{Lb#xwNP=k+2HUAVTO5Q$MK~Vq|}Qrv}V$hO9|`k$dUz!1u6s z0}1G=*7QtV6X1){UWjW7@s`bX&>6QbZBsmHnoOha-(Q1}(yvU|{8_$$gw-FG;Yt)2^ zQdA)KJ@%>+T)L&-K89g`_b?Vmh&GKKhxP#8oFyLSLh%Arg>w<}F{G;b>0RklZ5$Ue z#cH(acD-O4OlpZO zZW~}c)uJ=rHB3=nzKBUyPvb3Xb;cPU#uZ4rZ>6sC-VY_^co8G~@rTkD8=<}LoDZnC z&DKQ$hQAfB=eOa+d)YQdeBAw;_c&4)f532@O8CMyVP&)KB|6 zR6#B{Mj&|&nfY$Ai!#4r1}_tkY5@StG<#-xh_q~}Za!YgmSn<6CCz>I4uFIRRO!p0 z)>hQ}Su<)w9?f4o8{)SDgw|)0U*yPvL5C~8szXoggC*zg8htO5u@jZL&@X#eyAn$D z)Xo>TChUu}wn$bAwR=+sDG2!2@Ni?hliR3z3F)TLFOQ$m64qEh9IPcq9R(Y%83I|0 z_`8(4Z(&|LTR<;iLze!MVXl3LNQ4OkN{s;%z78UhmAoiyvs{$VuYHr=jUw!V5XkngXyH7rATtC>N{*9J7t`L*M|Xi z^Ad9-5+npbnJ6`Vb4XM!M{>Nd77ISPmpwiAa=?sz4jdHO4i-B09KImYq^JJ6DOnBU zbeZS-)3cwm;VU?92HUq~m!qzFH?+ zPuwl&ve{{hp%Svj-{gN0xjW~>d`YS;$hpQ>C6JxL#QB0~$3J9B?9d#kRaZWFz0aWYzV@-$2 zhaZKO-6(E4J4OmmJh|X|3;zAggFJbL zRnih{NYY5Zt8k>XMp2)11}ZDW9fc}{B!a-GnnNg7h(hWhvc~e4Ubd`1y(oJHp%>l; zzGc^Y-x0hQ{*YaH&vmDa*~_9f?*-ME>&-B6opwJqT?f|hiAu#9NI};-Yqzsp)mW9d zw4wkM)fk(6F2rs)Hn?O2mfVXQM{Kpy^wAmv>q$2=r_wcTC+3#2ymjDb}iLsq-ac)ft^{mA^Tj^aiYfzd&rf}*$)fu}z zNj^Vtx~A+x0<@+(0@JCosm`LN7klwahj_EZgC@H!X1%g34?YmDXvn+XF{_5w-Jj4_ z_a&^t4qX<0JG#N*1i)Wki;-#rxbJ#$&3St=L5IlKxWZTs-Pz1Ht2^Rg@p;Y6AzC1chj3uz2a@1-r5|U$fz$`Z|;&#ZeYQ?4GegLoapjb4SW7o=vRR#a{ zP&__5GL)d%wp6?z*pY-aSyBSE2Wt&rbDMALaMqRR!w0*n}YKB3B3P z2?%0y+foP6(>k_`{8^PcEhoFsxM79Jh3n<=VJl7from>DWD%XW?imo;c)6Q@Tnn;Li zdx+JFe<%w7@x_?je%Mt2$?bb&JIqZ+uU%IHM)9sccHRMjm@ep<;eI=L(-oTQ2#({Ke^y73}w__iA zxPLd>T5A~o_uT0xeIWiZ5}QjORry9Sv&e8_qSNbl{Z+}Fk5Op$U}LM@nm`oniU=^< z4X&-vK1D_AYWGzN(YK3eRnSY~rtEzCswFLITfQb9e(7BGG%|2jz0z?Sg2iNVxYiXt zSSDi39UI$iWPDB+IXHXTpf)s_^Fh>@B!$yl|Km_P4_h5m^N7>S!95xj}6#SP61^^WP6Z zUU65Lnpo6$(CNf%&3;RkZnbN^axku}lV%b{VJreBa#CZ>9LRXL#RM#FnoK)ic;=Qp z@SH67OgAJZ;(@dUc-#tG=uJO%Ihq=+d3>8yC9halIyb<yH-x* zJ~C`Q&LLdzDu=!QEzsPVo&#!UJZ?TzyJI_w7b*ZTEr<7mv&6b{YgWCKZH*~DS6r>U zwM~JW%R)&7!V_}LDY^-TSfN$gUEdsUMU#7@QB=?TmzRMG+U_}(W~E$Bk(n@B6Et#Q zqEd*Gxn#0E$Fbe`k5#{uthaC!Uv=Z z-1}niH9WgHj@fILcDmiZs%dHb0?C&v@KxW>DafR&VQ$s8p<-^=W)_uuf7p>x`XcM) z*@KIly7MGO}Mezhxt6E<7S_-=5;x^dzPy0)zpWu`s zxF>LPGeJou8SudG^oAOGcAKYgFk;A%*|gN@HQpV07c18eO>O&=$ctfci{iPumNi$J z6XP%5OI!*@JAb`A)Sn+JduVdcAhc5zv^lYDGA;5QH9d9Jm$BF_NQ)a}O9qN+q;ax) z9Uq+cFOBsV@QU!utW>;Pl9OM)nF*V&b^#>NFY?ob6VP1ITp%CVo9|J*i1?%m=9ciX zCX{&D^^W_AjOQ(~0K=sEZQtu>r60Q~6HUYs%R>TktauV8-EUoCA%+&d#Mxs^EDN6` z{RgU_2e{i_@4VfB2xQp5*zJbzzOA-g)X$dbFcAyY$P|rfI`&9|ue;ik?GI5VX58_o zjvQPRsvv!iJ|&681%OwIDWwP5<7g)NdXtpqs=baioJ^tk0amAL_amIY);5AjHsG`$ zJXWZ?F^*3_`vmTqcD9E)N<`h~VMZH2hx)4RCUAEa+7%ssr$Bo-*S_A=+G&Pf)R}op zfth!%YgnW)#amVSO3wDYony!9;$i&4^N^@ykDWIHSC9ICa-7 zhRSjX9+JiEP)!_oy4_R6U;0PK^PzC*(Z|Qy7DLmc3%+CMQ;Ux~R%RHFCipUG*2W0( zM1IAetFGPN1zR*FOSCq==?xea2|w2w8V#hEnS(*>2e}WcMMzucfv;ve)`!GbFNERO zZoh7Vt6fG&4(L}vqjWRtUb)8i7w+Bjy4mis)vn3?vioR$3xY|53x`G0hkMcsNZ+mw zxA`22u}+oZ++40TCeud_ZT^GHj==3V$JaIZDa5U2NdP=K9%I;#fue=eX*7#-5%5#^_2T*lB%>}U z@)>G@owjtO7<__6(uw-6sPmc#o8+E8$0|0eE!a*Og^&<)b46V`8}x zC}Q~;s4_loQj`cw+j3%yT{G`(+v7x(R+f)z2Ww=X4|H6jNJ;5Is^d_tLT|rIqZyV3 zMN{Jg`G-`}oDZckja}jE5~-f+zce%3f#dLP#APpKE|x4+(`APIO|Fa+!(4Z|BK)A1 zPSgl=beK8V4xL6ttVc}ittKlQXBQq?0ecPj7MB;ogK{rl#GqMVH+K~X1X&#RglUQe z6h~2NH?Fb{2(O^3w;*IHznrmVT7aInX`ZZohj#7(394KY#%nCR(DnT~adj86%##~Q zPu(bUdD~tx-suk;*e`c}k$dnx&F8t|K)Fpk9I%#4ZdLBIySh`e}O(aL(f{j>>c+ZmIN zHu3Cidd}IJHpbZp)uTKrX0YV^Ws!-yjKrFYdAZKEuR+R*+=wOfth{MZ_wb928Jffe z3WPnY`x;0%$Jeg`9y2f(K?FgONn_K&V&YMu9>BE7v{~=+^N&y0vuU56C0h?~ zG6{>kZhSF<61BW6b^X9&pWx;zdiaa6vz>4ebc>972}yi?0$IVxxTd8ehh53}Y!#q@ z_;VDs@AeA}E*|h?%D9t_mujCscza+b3Kmkjr(VwbRr6vD&Vx?)hrLcl!s(kW3ZHa9 ziUxo4#6@2_P2wP;6X6t8;|`0dQwS1LV_|pECkJQJY?X{(blcHor^a!L*#Erw75AC; zIn)Pp^BM6N?_T>9i961fRt!~;+Jt@A zPky<{S2tDJIatLiko`qE405Y?{2y=b?ZR}^qe^Vg+;Q_NH()mNZ zgN=o{Bpf4w{eQ1`_f%o-3ku6dNb;wp>hHce{SOlBfB_Bp2Z_N6Mm?v(lLLoU6iVa2 za9V$Ji0NQji=jYY(KD;%?f-^j|Lf8l6@|;ce#QPnT^Np+$o_A}xPP_lAI4vlDHMN9 zF$(w}-s4Vr%oNQ(5UfznKftiUiwBO!|7}Vk7W%BlU$$9Gp`$o0t9|`AIJqAELkaoM z&!i7%fx$m~98kQfq1XHvLL>P^C}zzrIo;>n?9o|4oechmkfkiU%XMapmh@4I;9$3-Mzh@&U*IXl{Fo8Uf=h5iR)+aKaGWsOHCw~9-7e**%oEAe%-KXrg+ zH^D#X%GBV(ezs(j#|%aj^wJpUd}<;imK+)x#NKxL25jaA|Dij$e{J2OF zf4jt6?H@!QO7L=&AH_KLH@1+FGE@dgl4;Jdp+tc5L!7|{kN&xHBI+F3TZ6mT(m0aw z-=Z;Y9r7n%|DP3|Jm}PA{~%eFUt)UDC_P;~VN(oD_Oln=ja(W@ID8b|?ElJUV+%!W z5n9O+#8UKc9>EaiWlHqHf1h8Ifn&QlVzd)~_JO@)c^WUqxFh9B6?aytJTw3NU2s_6 zWx`~~zibVU3Mu*bBSqz!6c`O#^o*QN5E*UROsX_Qk(D`^Ey?p(#-q&tboEfq*buS5 z#=+miun-;n1hk=gD>Q@mhcd2-26%VEhQ!0uRm`X-_;=JW#1z&0$62V6ijOYJER!UN zad8Tzn&47Xv#vn1+UiJMj-*KFc%$>rg-!d|4x#_P1**qO5a-kT9j8UK+dq~rvQd^$ zr6$@8%-5u}4i9+4@t1+1QEZ3H|9s2544Yl=~h{tPZv_zD`Y@@$Z>(3?upe7-v1hEQSU=u{U^h! z0={yHKhvid+4MrYari>;kj`)H;x5KQI+))y|1#+>6372EjuU;6IG^%8iYMQO)Xw@m zS2CO|Mpzkg=AiUfW1_!fJ52oJ_rJYOPlWzMvn)?8e}&;$jSw8;*Cb1*F3;aIb-IlC zUjH4}lj=nRpcD|%Wxf>Q zbuT>+S>F5ewYV}$5u2{(>R6^T3+4OqkV}uVcOkB|m2}=r37h{{-oBH|G;1p{wkQsV z_WEM8i2Y7Z^FXdhZHK+J{B8k6d2f|x1}CmLHmW=aT&4!VZI-!JY&&|kahprR4> zpiKAZaZn#Ip7bX2AJhMBZD=W5 z{#L5Ok3Trx&r=+)S^nQf7D}Uig_1;v{$Y{DkT%Hr^#6QD=LGTS57ceLSy6ZRpe3sO z|1=_W^SQ#?D$JBlkw5&e_74o|^#A2Riu1Fd|KtMx@$9lAw!>?b35Cq4HaBgiZu?Z) zjE@a$HrH`v?mICh1VswLRY(Z*{{0O8Zb&D4z7OhM)~u*^VQx>h8@3%@jtK~gkC}er zfzbctF{D(J@1|0NSPq=%{BIUrLNY)L^EqYOSSOI_Vqge=nAlwwe|MEb6! zrTYly`(r1NzGdL*6?kLDmQNE$88_75XP>FQ#{YWK*|Zt-@NPX=8HD#~XE&>{n}^=m zBQx<&?^=fZ!DT;q1)U3`aZCcMe^hIp3x{%8 z?$tM2O~x?@x7P2)O1=ymyt~?U<^0gVSZUfq0;Jbf=|lt=(Ou06IxW@9e$nR|_LITc zc;#hMD zH4MD&;kOx5*0Unz6E|Eaa;#sO_G!V&B8v^)G%gEGY=iy#fp-s3CBG<+bdK#9B@S8N zN7|5a|5ouyfrW#V=J1^C`)(i@5)mkU7Qsgw*<-cOA`yJ`luA#VQnvnHFk8ms_m(h@ z{qN)`ZR_9tjv1&^%?%Zc)JzfL*(%{&XKjn29LN@%+sE;DHu3dgk!VNMv#wOCH==s; zp=MaD36x9a9T(SAx>S?F5-`Ht5Ht&do)v&5dh<~r&BZC>=z*4R%cNL9W7fBjIpYFV*!O<3d`*fxbFy0I$KX z&=g$D`MNvT zG%`WaV82bDEp_FTdRO!_@_3LBM?8Eh}Euj(VmHPRsdTWaHJDavYq` z&GH(mM;t5lfvk1T^57=L=1c556Nsi8Bk@?SIeDdo=XXDpEHRd3xQJ7cP`+{1RAXIrhFk zNjhqns7_|m6_FqtPvO&NGz{H8mgOd|h#!|6QV-lo#o2&ocT3YPB{l6iFLlPM@UV`2 zu!wqpDaeOGb$;4sF@fiC`DugWBjJ$E0;hIx-+GSQ;f+etT&)$J+~!s2+DI$HfWa*l z?fCX=Vqd|yVWSEyI3^(P{JKBzrXR;h*tP7!%q#u}0Q(w~Nq+EZcsOBa3jzUXC_ur!U8ZZl7Sg-s<$xMFtd0 zYjrP>P9&fr-pd0hNlUqg5Z$pz5a-oR>GF!>Fp=dMvNc( zZ}Hhr^Jlxa<)tGmVBt)p5PhIznc(9~7i@`D{!p~^Kp-#!Il8;BenHd6S;BC8?SQ{2 zO;p!6(p7w%P8C1UU1~XY(yMnT4+X>E1k%OkVbDv`xe+I(2#L5pWBWz@e_S7 z;$()H)VQ;3VREm9J!94okm={DlJ+K1!Sb~B+PTT~JHcnt$7}Il$^qBii&04u#w4s7 znd;+B+}h09{5yDXOwak6$2%CM4(aA;?STtopR|&{+sVv5K`FHg!!jD3X)U2M*#HFF zHmy==LiTqUIgPvR67j|5mDQld%kbVN#Z~ma(uH6YgM}s_k7vp<5|KChlvIi_b*PkG?J+DmR&0Q41lFAF+&uz@h`T#!E3Y1j1v^&Wq$7`K06 zA1@N&?y%3~yUym*HNX5NLZlCIAQa7`SAu=!a{6eFl=UgYUn`KYKaxu5Qae3#B4O^^ z6aj&OzqTnOMT8sUc_%Pg)91dGcjSLpu#>Rp3NVuK5%OyY)=W7)wioChzgoJ!P2dHp z9#8+Wx%e_QrJl*lwRlsF^_W2--rV)%V*or`$|mJLN5To{j6(HCM|e^c?nedcwghm| zN`f$#f-l4Rcjwx*Ujee^+^GljSi2Dx_wf(wDD#cBtkpjVqUjYsiy*O!wfUy2@#szh z8=Uz>Ihg`1bXMiA>QX>O^Ess+R4jS6JHsK;gG0)u%w(f8-2^O3>B4J#8c#3w2+5tU zRM#M&8Ti>r$Yq%@Oxd`~v`F|xjszPl-bCE>SA`>nu?+6?ew)vfIj!uw_-CDd2#6GA z%tr}1QxrD=a9F8EPo>MF4@^b4KGxxgWhw{@H!m!e^lQ4l-92^~? zlPhtXq-xPw2K~_-uaKNaP;1jk8>B2~84GtI_9N>z4(?dk+1yYPq#p*xFNatYM`z*Q z)eMjixCC=U20R5ng+$@2&s%4ro|}j%4(f9$MC9@)e>U}af^&@wh3J-i91>RYln=Np z;w+~X!2=3llD|n%^f84@q3wrV_*_Aod{^0z$OoggNP(6!-;plY?+3Bxh7Vs9Dg3T4 zQwYN)pp}*X9h2OrU4Xc@7{sUQV;bT#ET&xn^@f&r*tJQY58VePo#~gGk&}XJLXf%A zp7iJYOX-Jhm+zG>+txjw=gz>Lvl?)^d1q?#U*~+uFL3VSzp#D@}r%vSYW6meGkuVDPX+J{4<$ggnqBwvKh6d?blL zkj&+W;o?2ARax`*plEUq3JQQ-qv>g_!@}1m@EoZqXVqs#!{2!1nu7P~+Zp|r%BOnv z-`<@aDtV5Sd?P~X;Nh%BFwB?fq<_&TA$a{W3Q=5+Pto}9O!(7C%Hyi`+b=)DG(!Qu zYzCV^XUv)HceQKZtQR$YIKHB}5p+c?I-7q}ksJ02xcAb3t4}Zl6c7)lq5nSBM=9EL zDIb7Mtt=xW%X)u-vl>wTyzhe2hHYo84*;%+sAkz>f-z_S@#BtsVqs@_X;+fnuHm@oXfG z!&JT=>xh^a4BpDwP3F_JesSSTqBas1V{Ii2CZV{TS4bpR$qR|H%ves@B?6WSW-AFk|oBi#Z++c z)$ey@=t`!klvIFaW2v`*TI4|FI$c(%mLnNXw1+QCo#ZO8$UM$3I-RjY6GBJhz1($+ z^_?xaCLnaxHc3dwpv(|p=I2Fmwv*fzam<{fiyfCPkWkTFXN_z#_~_b7@0`T0L~Wy$ zfx-rR7oErp-u^7{f6MwTjY_oQ0OH&9d&(d7Ab4ll{ga?-!nlEESghC*UWdqAimxi| z>t5iG7E@HvGwUgs87fM{3D%aRxnZb787Sc=0*Z>LIxUzJpg0(jBNN3Yk{%Kn|C9|+ zDlQ~AuD{TFH`ZaX4AQpU0sElN;`T)CbUBt~}3kFe9o< z2XgkIx2HTE*Q+Kd>XWD@ze{d!*CjeCSr@p`jo^hpuc!tc2K4rp=0xyOIKQh8?2x5T zr5Pw;cCUClwq2gMsnU4f^6U66k0rR^M#6tqXuBtxw%OBixMGaz$*!whW&mY@Q*muR z6JsFk-3ruIj!MKM8xkrPc!VFsa4G#p{4-mueD_=NFJOZ>!?8HUA}kF_(WmMm%{nn9 ziiP)`K^GU>iMorNp>?HjG%#n}y&3Nu!*qfbY1Dh+>=>ber?(?H^5Pv=WdgH4{?FI$ z&bOyFuO|#wpl^WFk{Cl#qK+?Db9IfzKWT=28=A@_Y;B~9Ux#fTs@w~VF1T5BhPGqr zlq){>)BSA3v^Jdl@}`nslaf~j#tVp`nWHg{bT(@UAPoq{MUMfGA^S4+e-#CvHZooraOWEG!UF@`K zch$569zRAMqZ$~n{N9iA*#OEOsZGtUQLLPy8BlZ~>iknbp2d*gtm{aHN2lD1kM&t5 zCS8k9=e0#!;t-|4YJWkIDz$|BN%*`k4q^#wFk=0)4k!ztKDaYi!w?|nzKJP;XE@E7 zkv;QOzx1Q7OB)x-7aogpbW~hn!mZ~6-Hf^qlt*7GJV#b2;6$N!8$907&QnABrFIgo zJ)t)2V-vadS-m27Ht3AaHStNgWCzk|EXS=ha}1=6*htcK8=B&b`k1O=dA`1o|8dh5 zKUF`HwPvE&V?jgRNv?i_D;sMner1>JAvZN|T^qzs%N97OX~?xpyF0$$zF=}>(|RG! z=$$& zUPkqp3Ia6>Dyk?^twe9O%{CQ}1Vqjy+$yR#4pbCJ(@9gzQnO&rDW(;dr#ua?;Ve7$ za3pJTiDI#ckEi( z3S=4y1=lc#PN8>h=S(SOuPB@t`X1YDJ>KzHs&WVGetmPxE$q#ysTM}J?R7Y>BDBZ) zA`O!xA)9Qm>pJ5IUa4@14pT57f2peV5RBzy1Nlv{)Ue7EiL1jfm+^@M%{# zyg}`)%Z5v&`>@XC11^a6nw0OP4=(7Isqo`sJk4e(I*uLVh4bVz{YH&eY4_pUzSODQ zla-9jqFNqETm*l!`)x^^1<9{>Or&y0fh#$cABFk_16DPn#=ur9!%Lw(uuWx_GMR0a zKZ!4*=YKMR`Kz@(Vj`13>k&W8t`-K=I#satBRgaJG5G%TfXT;HiwKkZp z8IXNh5;aJ^udn9|wE_nFmd4RZi>>s;C<&}QTUG_0r;M`_UA z;~eD*>N3YN>8>a*$KnK8o*kT|2*OcySi!nUEJbLe2b5NCTWHq=9cC-LTUs zo)^HvL&CWMkJt$OcHJLT3YoG>W<=oAX03h%H=`pE|4QX* z70E2u>zw_whDXx8R)v@i;stS@bVd?0iK~v z4aC4s3cm7kGPUkr4kTD)iJt9jPjsJqxCa;#O561DhVQO%zc;Fp8R1=FacAR6Ri2H9 z>AWQyYyq-E_7`^!Jv)Dr!+j;LZ!52vI!KeFUT`$`9mekRzBx%#Q_K~~fZV5wN)2#UG2zQ2Y0f&4nxojh&X>DSGyT&4nV6`)X|o>5&Z0h4xKl^)$L;>2c&sR@}u zf%HhvCu%Z9WcG-3v+wUMBxv43j;QEg`KB~e4C|1HE1EuP^%A9U&*~~@kFukAZ2!&? zf7%is9Uf!UFoKd>|Fh-WmC(o4rjsS8oG-E(k|eklb-17@BD=r8pm`zaI0Psnzb`$$ zk18$2>g`cqBOZm1Uj4i?;f+D(&)%}n6AG59WE|-G`iTEZg7Mi06O`cg8icorM43LD z-_~RL8hae55hpdW9ip*Y`8<(Bg!8x0h&GJm_MLYt%5`VbMAYfET}z=eL)mrMD}-PJ zTFU*9a(4#SZcDly1BX z;*k{8BMwxn((rc;(%#FT+qB{L=`tfr*p=}!_12_K?Y(Kpg^3B$nqhnCtDk?B6AF3I z`hvSQUtgX0=csa?SqIhBiO6^Q(w(iXAlz-AcMWv75K&&OuFKwufXbAM5?+ zVI?||ZPX5>eThN1d)V$;A3rXg8@BA$%7T?>`TcXuM?(1&Lva|o^x;F1x-N1o)0Pq0 z+mYEP7={XhxmW})rgg;6UyJ=>dGn*3|2*&h9M8dMc@P~(Fda5RGX%jk)f+*c$XMhD zP5AYH;0oi6F*TQed*l>ww9esihLN6V-=DJh3G)tcMc8p&k;faC9^z(>OU6&cXFAi~Py^i0f2GK2t3~kt%ajgw6w@|6S zRwsfCDHrOXo8M|IbZ1r4@Eb{jtZ6rV&J45{!ndEmGiNh|32oK1IgcnUT}AZUS$y9;G{Z)0R$}+uDVZ(mu9zM#V z$Qu?!+w?x2eG%%|8NheEv(pnxs_h3Fqz(w%gxrUe>oz#g!dO>!&Q z$MnVt{(+K1@;)oj{xs=w&Rj{nwG`y`&GHc8L~G-Y;$xACA>mb!Z)n}Z$-a+NPBx;_ zls}pBT>H3N3hXT77Vk0el8usWHV6oaNoIRoIIIKDk#5=NY>EicSV$JOqC=&X9>BlX zW`_YAGJT|cc85(^1wK>NIq}CVHCshN76L;KmDs6@87baCP4Rxx7~l;j`^t*eHJ~N{ z!@KO|k50iw`ML6eXmaU+oyFJQ0S-PcloHmn@Ko~opiUtQ0mT5fNRzA8Bwxz-l?LCR zz{VL{MDp{M+KrLO@l?&wQ)vn~+oFChDvx9hJlgIbmTNRq2X^rfXx*E)gH)a>@g#20 zkFse(^lj~AGQ`}tpqLqWt6TBn2U8QA4s?soiIZj~1uCWM>d8z5u_;{d3+>{*Uvj8* z&`vil%u)H@)VOZQ9f;mcaKNM0yC(@cEwf@=5;ZUF{#l$VGPDt!Dh9Tjwae&8%{en{V}}oY&B+FxAbL40;WFUoO|-b=<)9>rmi2 z#Gb_MC))X%_Pu}0vweqjzSDy)C0ZqAf~dHTSk?u;%%cXUT-b!ABA!TqUuES7giiiMCmh) zZGtuSyLK`^K}*Nn&rj$r(7ER+h1yGZWIX8==yRRMJ5u1^(Rw{~Bw`hdjFDuCArR_k z4i;K>iS3DVW*u*Q+gzu=T6{A<8+y6*cCO|tAMTJx|Lbb>ZoSfHd{K(E1jqJpLVUtK z$C(atsWTAWwxRO_0$%-DR1)KT+<>Qpv#T?iG;ZK!S9-NQG2f8&2ZJ zqKPiwWNz=GU_Y_nfowxKbuhoToX^%+#{N??K?H6U8POne!c|^n0p=w*LM7hxCbW<~ zGT6wJOU>Sx!gb|)RwR$OQ80DAx#JDD1U;myMoBHH$}I?hv2=1ru{B;w@XZy%-n4z` zt_`r)Z}CkFp=03j=6&Z^SJWnN(RTiD{L^z5-88#`BAg8|l3bpyI`t;w*<1T` zG7jH^au8gD(P!1~qhu9)03>noUA}v}{>`uJjT?hl_ZBaYtvP-pU86vSNiBUhlp$du z>pU9WIWJ&OxGZaj`)677<;(S~76P2w=S3O1o2K9D)FI!6!(=b!IZr<=8c6~;uH|8DFLLx-*V0YQvu<{rm((7#&iln6_+98cIXWcjM92-Lld-33lUcX1&OLb z+nHY#r{}(>sW@KaEZfjlNfeSj7d@pvS|Hle<~F`YZ|GuN=6GAlyS(Tr2TLHO*&Cdz zw-(vVaBZjX>XJhmuI5MvQKUr`1;8eR1nm313N#$Y(x~C*8Pe z3BH1x#QY{G4}Anig(MqwC==CPw}mz5!Qwif382XUSOj^pUI4OfabED2RCb5Si|7=* zdUz1{WumR2Ul?(UX7O^shwmhaK#$M2KZ6fL7>Ee}C4U#ew7$@^x5{z5Rhh`9+lh?V z^&8UX1CPTFZ;_IW9EsWZ(WzNZ_rC@)D;uazMl6MJUsV`%5d++c(Z0HsJ*7IGlV6^0 zMy1W2<0K^i$psK$0MaWL(R@sEQK}B~?4ji1^g|x;bgK`sDkn%UUbb1|IGJ`32ViM4 z$bwhUf|sbNiau5$ysWxBi9jE0agIAYOhFLn5T8`z14+UgKcY?8y`-WbZhXh(q)2n@mS+s(yL=-chX<&==)5!yC)CsC*?5@ z7&B}#Vv6wXATagti`%w)d!xo+z6RO_d^o*ch=0*}?h=OoOCFS9-@?%jev0*d?K%Q&0~K08``YL0;b=A zSX5uoF}dT3^MAOxRFLD1+GV7o-g{L}XD|-`;s~zO2D-;tLVp54w7+(s{cQ zQ2Dy`OF)y2G0o!&wUMhQ;84pz?7oTn#>yq@cp9$iciSWZN2h&-8S8u*hDX;vyYrnb zLX7ndQ=bhnq_uZHbO7g!5?x@`B(Ld|R4#Mk{`mwq$Z(oHD|qT zwC?lxz_9JRvKd3PThxW@T*O%3@w)s%9&YkY>gP0>4cQ=*kD)0*$Y$o9)@#8e^8@@x z4N)5)!A^tBAC51FB#J+30i-7LY~0(o1QG!2T^%E7EIgCDCUK`&lT4)m>B=L|DSGZP zQ7yLszPS$k`(diyLUcL=&UAona+_wb>bA|(%w2y!NbyQe-B2q6m9pJjh=zUHt!sL9t|;hK|FyJ!3TKc-BU~ZjKZk& zJ9iUip76tX;o-K|K5rv(0~eJm(Io-#E_v@%A}atr2=Ye+bD~7Sg`Z*$;8U()v25q3 zKDVFnX4_|89NCza|0u^#Q&GYMGp3}5uj}>{I)&zn6ITC8{cQ@ z1+}uo@zUMyRHyK^SQU%obXsNPvsG`fg1Il=M$$q095fu>F2&^4qOa$ZanDdTUh=+Z z;Van5P+#n{L~z@@11Ra&dwOw0U>w9xh?bt>s7JlN&{c9d!^FClLmu&iG^4+-38d@ z?mb(}3#iI@E-&11+gcT>!M#0yJK@4~HO)$Jg9Py#H0v*mTu1F|ScV)P$d)aA>4>LH zz)MRSj8`HhUrS$2+}$}*R7)va&(O&aY4t(TVf*RGNu=9fJ`RtH8F`ZDOF8?_b%Z_I z=6-yXnGlVVI%zuC*}p#NJaSHkDwSR7&87c1vIxGcSQdbFc^>Zm&QYX9k@7g#=VsaK z;1;w!lxuCY;y(h4bg!7uR_kWGZ+P-#pFZz=cbf3dIln6g@%!(>XKIMryRR2@pMCC9 zE)U;sS?Kb6OBywfaL7{QXVaoESAfCDS5d6bl0$Ia9fadGQzV!!zT)fiUf!_6CnhlT z*LjD2%y3}{2nZZ7WYF~(U=4iasADi&ocN?k$_lO{GNiEWPeG#x0*58DuQO&IZPZe^ z8mH|<5b`6M4x}2uxT5kW5gY_VD^UUbZ#&jkp9dj9>@pYk`AtOahULDxE8Gw z_}!r`bcZSftaogd9F(>;J1c>qZ_w!oY^(l zO;6vKQ#4(5y`a&p)S_GzrHzB1+P;>mU75sR@3vpext=uhcUfptaR;P3ju(sN{d}L` zsp@eZ)_M+J6$S&Q(#S>WmL3Y*`Poz6w7Vzxe?0$L%ZsXaPe}^QAq>b(&Gf1r6v?=Z zdmg7r`?|Lcc8fGQ3DmX2XSxA8hj!QO=^t9EWn|%=v57}F@ zC)j*$aC4%@2Qsa;0k_jEnT9#leo&e+wJ0`cP$s_{36%FUMsfOu)j>iC%7ylk81!Qs z!;+tFBl5AaZXSW`7oT>%Znl=Wo%V~Kpt`v_SG3q*rS98a%%e-L97(gMvlidOUo8-h zl_5Z4xY^lB1%2lvk*E=Ub-1*BsPL$>DQ{jBJ1&D47Ew1)z=Kj*6cwe2qJSXoa-ax7+|SHw~|Tq-PO6H z)H^@}K3aQ~SdiMx`HopR!rUt6Yfh#~cusgxTAv1c#4=$Bz1PrQ>34>gdoX&5rSyeE z>5t7k^ir%9X_`0(Kt>U8qSjdO5)W1zA5Enkx6@mmv?z=~0dq4Uc# z+k3S;NjHbb8II+k`AWl1-E!ZtR4@W%uOY7_MF>q!??J&fB``!JG9ARE`nApZy7wCg zLyG%{fM;L~iF|?_J0AU4V^{IO?H+-n#TGLCB(DNiP}&WwOKl1!elY>-0mdTgs|ccH zWvbr5Y_N8L#;fG?p%@Uz@WUdJehFcUw@c`%E|q>XUiuD|!A@V9i_OUgv&7)vTVtpt z`HzSjfJvevr%Jc_91`1m`(2==4g1jW@BEfbt+(wQ9p>&iI8WDSAU-$BrTSM z#yQTZD)a#Aduajbw?&QrsZC&pSZmeUIq6i5p14TLr7oslWbz#oX~?be&?7<8rO?^( zd=36Qw=X_!pEbMpWf&AocPt2$H~j2yD}fnua`JXkOuiDtjnvHx9 zypJ>EZIltOpu*K**Z7I2W7O|dtnaZh@Llz_uiG;39g9XTnL6>k&vIx8**Kl;7vRNk z@d2!-kZa4Oru~`lBHBIpRxQgI!Q=U*zluJs;~6O2F)q7Ueav!yeQJmL&%@Q(y5&4s zi8tWcObEQWJVe`oAsy}QBL@ViO*?N68H~s*v&t6!JpSb4HT+SUtvmLXA!S1*aWmZ9 z#hC8g`%KLG+NN)%Mu3ySJyEo!(0^0l$ogik*An{pzW(Kbp8&zSG11lhMTb8j5?-yb zek-lehO^?_(@esws!Vd=hmcoE@`S}b_2?3LAYDLk;N!8BX5yp)59EXDj8V%SHp&=D zXS_mW*$M_IhlOs*{SGzwUI1%W&^8LO`l%6cPmXg@eaR^Fq{^V#Z~ju`kB^XKUSEiu?8S zJ`BAEkg1^M@%s$Uwkp_A_n7XhmRO&Oz=h%2k-IC~;b0Jbs) z_Y~v6%?`kfJhMklmb8PmB6uQi=4y)v!o4_R z{yJ<`@I^SKn4vU|7cD}IX$_6 z4mvCe*YNflOnD@?h(IRcfp|5DB^K~-TX_1+?V?lHiWK}DAcNckQPD)j>2&Pa!;YuQ z3=$!Rr3Ot*aH=_Qi)4skecmpR&90hWu8e`f!DhLra(qa=P$ zt)%lm6`p9ynL-iW926+gxh4S!7wPu}2Xd)%Kdo=r{&_N=H}8Qcxs*W3 z^mEzjBcjaOH-=-H$Wwh&T#rMcPE+V_QaZn8oh?P^3P3Lt5=T+DSxlK3p zr?zFWXZpc6v^BW2kPhA9Pfjg{5_KU%V)nC8rL_g`5@vCq8_?O#F%N}?<(KH72UG~* z!!2AzU&y^Jt_d^9mja7()|?EiW7sl$^Og*h%nug8AmMmi@%QX;^qbvX>`ixAI_jg| z`(z%f6&i!%hID0j%*qsn+&8Eewyz>PUF~mBwb-2o^s(TsEJ?WIyoB98cgIs_f%@Xx z@ULqfCpvyNraEx zQEC2+r!;;JW=U0MH0+uCCe3TJHzfOM@R z{DM91v8?PC>#+f@kO7kCO2a(&{`4+U;>`mn{n8G0(^df!OQIcDCI`uc8&mBL1i701 z0pQJ4_txY43cqP1suo39JT30QrxwMwf3pAy^y;$LUiLaT^K$bNF_$Tx$M6yy`=H4C zuQ^P;qWvdA-9n{hhRv+D5b7DG=2-nZvnUvs#2#Cp*m%g4TE*o;GneX|SA+YfA%3XG z0fC3Ss7hvjk!qD#pB5VuMd1}TqKgI#5ZJA=V~d|sg|;FhiD5>J1}sL1pKP``u75WR zHxzQlCk$5X^lwILo3S1GKs)EG+Kw0#FszTz2FZ9gG-EWB=Nyj=!|6z;j{n}`N6iR) zJ2goz4;GvwNglFah8zvs=d^iGav5-iF8#(=ZzDIt=O&%p?93)~U=+0;uz^$dj$cHu z=uhl!-RD$>2Ei-m)2ge*0-dViU3`&4%2p3A*HAhFAI5*e0L!jNw=A9IdCA4z!=u)jWD%u`hkuvj0# z)lYYQ?$12cyGi!uYe@eax*0uFZ&6hKT%}YwS~4G#SS1qQH|GgE*=qmC95JqFaUo-i zITcz_?1o_-0c{WNI<|{FqtX;|09D1h-%uHkbfsgfnI{Gc`SuD68j~})RDOwELfy2d z&qV8+W~uxnbO0kqxwV9c2VKjbsO1V%*z5D|iS4dl2IlpH<>i0@VUXE<9Lfn*h5~>B zccMGrE;%lH9Q^hK=rLyLckWKU&Av;aPim~@vK*l<_c1@#8~8o1h{#EAYcRa#RPQfJ z^a3`OM}#T>8f~u0M|(`zXqwr3IO(!dxWBEU&))RQKCxUlx}oE$P*{=Ce-uI_@g8CI zR>zg0r*pgy0Kxem2xlBP#edDI!EJ(61HF)Foh=DBY?D)V_`<#yZS+40PEMm)kG&%v z?}=UOBkmSKiM@ahG@P6%%BJU;p!}gPE6&pc1r`Ho_?VmQ(u=q^-(BGEgq)7h3pCUm z77u(X`~2C^V{+dFXSUpv16-nH#aLHsneVDLZUT!s74YBP)=G<(tyUFmIrg2W`<{QO zhl9urDZY%Pj1SiQ^2YokfjOF#7l=-+1uD@U=P-CaPLstI>EjIkX=Xpqe%IR*Ab~{y zZgPs~6LMXNBLe|L%jc>*Z1dxu#Z1#~p-NIYTxvpzU8UTwXaYR`uBn1%G@VH1>Z6#clpnK7E%bkH8wwhdPkP zr+%G$chbs2tr4u7Cz@F1AYN`b&a6lp-SxnkP*2REOLWkOE-p?h?#*H~l=(7*0IWdg z*|)Z&O+Q&+q#pO?1VRgh_OI4XQh9T^Nl6<9Z%Mv9?~VDppbWNsYMXWjUs9L}9?$mj z0^x9GbtsheC*47MBx2+a?LRU|+*_x{??HYsK<32DC`ihXo|8_Th&cnQ*<5 z*;+*5k*!}YZDXpi*oI44UM%QZPptaFF2HyO9CAePiOn6~&YWH?GC#zY+3y{} zS-(K!kD8WtvgY7Ua=u3 zCWvB7=)Y2{P(~3Rd`MDTili2@AS&W3uQhb=yq*ZtZaegIXVj~eF;KNyyR3C)3n}_| zoG#&fi*-VAe~TJmkm|PF%p)I-Ky2<+DAIz%DYLWoPH+=*H<{H|UL95gb)3hgOteQl z(plLnbe^tq<)Wyy4yRE1;4go>y_!F`IO&`fy%nFauiB#NQ*Nj6Bsr$<88(Wc1xX!% zr($8ayp3UQ!#6w2s@R3{q)nRz_nB8}^g3={Q0q6jpa8j(O#Q&NY9pZg7)&R=JwC6C z1gL_(T2_azVjXD)=fM+eE^yR5ajX^qoZ*r7$g!+@wq$cZ_taVKhl|Otoj+r@wk;7d z`B?f09wtQ;><<7VDYbJ}I10~E=9Lu$ zCcElv9wzO8xd1q6nJN}QId-4;X(I=ZG8~Tll3BX?%kTo2h%JEXD<@*)vL(JAx)sKX z5+l<5yRUnk1sQ?)X3;rZ*(q+_6Wz^e^TyuqF7)vJiiFk&PA|0DldLN0 zJC{~*WpZ?>BjlBh>~nJ?xMhNcZm@6=zA6_PBwq+{asPoqhU0Es7z%nXJUAb9UR31| z%ad^lJ8eE`$+Q3cs|9X(V(`3w$O`>UCQtg+A=&Ko3_xeI_xDuPqtB|41+3Y%QB6SU3}-1!J9L^uX~DiGuGD;GWpk zN)(%mH-T72soW%AjuFc<=$@Eoz4|FX*jL^n?wG^FTanq5ub=Bg57tDMX+sId-ce9uHxc$E)9H$!)GLjOtETewNKO3C7i?&y!q?U&rq< zLfaJ1qkkWn>S0otOr%#P1 zVdAYX=8m7pr7vr0W+Q^njbx+UHvTKOqhY^oM(f~h%ZIqvr6U!qo|oKJUF@+{XI>GA z%3V?Y-I#GKfSd00kTBx;nzYPpe6e;b7JCb>_^*#@k+Z@88!+@;ZH%RekL*W(9LW@t z7SS5(H$uLXFOua7!a2rFf_+}eAHYBhB%7`R&7BrchSNDxaj8~{h(%2m3q!vmI^TiD zB)yHL&UV2P30DQya*}Lj@s&Q$x9DTJSTP-ZeIQXvi^O5rmm=rem_JO!DIwj*0@JJe z1Q~`KZ+T+jFC_RJbDS0-JY{{|4(oPWiwVxniNzX-z-ML}&Zd@)Ja>njm=mLtqut*5 zv=a(*a}-6HmB|^;DDf&cnUP)NOnPK8Vr?!Dc~^5C?2b=?YriOdl|~^kr5%!WY)zxt zVAeUEzsnVgUuqhpq}p9m?(khP|N7$Re^#Fr5F=HnIdcq)TiGiH!_Qx~{x|**mEKQm z5RE(dV^>%|Wtzk%)lm~VNVDgrlkq~~r(@sM?p0jKoB{>2G8HO=p`;;A1`U1lLBzk{ zA^L^-AWPI;_=;|U<*Vf04c;rB4wz|oC%NVCMtTC~x8bLooNrh+Ua2J51f#v|h8v!u zU=z@6AZ2<=dxO|sgNHUlqgY&l0*om)0i!Z#tb$yY^HL{?YkE3tjFpW}Kb-mHcvBUnV$LAhBup?+ag_V7d z1X)d{i{L>$9uhcG-dx`Pn3l#9%qiDMP8cj#c*2j1B-Z!&1RT!vyxTj=>=?PY)0OzLZNh`#n%qH9R6_ zGc7W7k^i(*B*dU$hYf>10vTx8A;4sTh_f~M>@l{01Vqt_FQ+|M^c!1yD-(PUkbMrG z5|E(PuBfm*IF)?OQ`zOQ&kn6*ctSTy-~_Y4s<{fq#j6_$-o6b65Fy38;|)d82+I7( z3{-6cH6C-{HiN1uBiU`%T!-ZWl*a@r&e_SB8h2lqy*m_WB*^nZboI2JA#!FKJ|Ez`#8yY$)zCX=K3ekr< zKa1q?9Wgee3A%OSZ>uhva&Q=?=Xs+#16k~hB%I|T34DX*Lnj*S;UUzQ4cv>!?f4qM z?qHYsyy}KuW`~GHZ@xmLo!aXjH@b7fX_4%kFwOii_p=j9wpPmx{G^_ z(7-^>Z8zwy%U5X4M<-(+GXjlIAXhoF4=2V8-)tZ&z_P0aC#L{8*omI2v^2X)Y_4Uh zZ5<)hfR-dMI^8);1dgcq%I0EuoAlBb$8rQ}8(z$pH=4DkPhH7rm@P!2 zsOuln%{?eoI%(XnonHkbzWZm%^H}c>k-Up* zF=y%LlNIUYx~y=XZPUs5@wkBb97c*0w~U4&{+p-jzVTt<5I5Omqtb^pxJ!Z9%mGSW z%cn@jk3SG1{}Ph2-$^>4NTW^VP$zJCrarog1zN4kPY3m+6>d03e4Ox%&0XMQS8|n~ z`!@@400$r^DXeZpw1H1M9>{Pw^Ecs$=Y#CLiBZnEh>2#yDPokFjqtB&cdg`$Sjl6l zajg$_Bv_y^}ag;CN&1bTNrHt5=x+FB(->?wfI!vXK`Qi z`QC`&xU!Ev#NX{Q5ac=3q1K`iEW%C@w99oBYC&uX-*bkp4wnNVb;s||`iR-O2FSA7 zi~&^8l>Ni+MtnS&a*sdXPM_C9$)&heo8ty@O|BHugBVl8})*F8a@_csO$ zpzgBdtE(czt1~^=3^7_0?mK-d$n_y=@jGM56QW+#!jFtIs2vl0NWNq&=Ns|}-3`vA z3zQp!_Hd7uezfx8XW5QnhiJnO9);%$G4lcb*Nv0={6F|*ahp>Kj>9?OVe7C`r3SED zQpmC}X~&`K=DoESETh->UBv2gdQQ!z*K&;r@%(nD{ryj$MHxv2&U4pS*c=7@lnWm z&FtpAz*Wg0thE4nF%BBEKZu|$rh>tunT&F`Qf7wJDi zI8z{orbHELOSy6Uv)GjTSfi$ZflgkvGVMlIW*hpoPGP=Y4-b8ld+C zxCu2@OKDC2^J4Q2wHP0_Qa#@mg#(4(rF+9y-iz(oOw;ukO=C*v?fUV%R>XQMK5D;GV}57xs`9og=CG*=Drs#dX z<5gIq6o%);OVca0QgZ?zyOP~%{8gf4ew#@=IL2n8!Lzv4p%CC067LQOJeIX`fa8uo>g$CU+92aytUK3K zA)L24^ie(+Z--Snp>oR1kr|Rl@j^5SyNOS{uhX8fEU1J#uWq=(qK?1hV++Sk>r9rk`_YfInCb zojP9AmGZYs%U-y&V#pDMH*R}KyGgYUG;=MNl0@E2{r`oGf!9J->Z6RZ90QJBG-OvP z*FOL-+S1g%V=sTAk@H3Tu?rvXMo%JP$2BhOOQ!6~z*o=`JNv4KAwv9HJJE7>1+17- zbI;)5#N=LDkTDjog|iV?KgMST$+^i1y>jvCfZzmjEALtlf!3@C?1ZYPJ!!5#s@O64 zNJb>~yyQfu?O5-yHlY?*NdabRc^|B1E>(0l6v#OrG6i3(ElTErfxN+lGR2832YEac zw0<(wyl5GYcTt1UUr9zq56NG`2*Idklf=0PnT11yHHr;JIlM^PMJf-2FDQed$1Bg= zv*+wQRmHzxvLpp7=TV9}q=hp_w$M2t;HHgosWv`gdf-Pvy2j~jEkg~MrS-UY0^{yt zKeannh7qG{>}Q(Qrl5j)$={3N#Tix(n2P z*{BTwkYcyMFcG~T1>cE~t&v0AQqKs$1)RS}Ci7I1Z~~*_>QDiCgIoAM-iWGpv3@5L zCl!2G^Plk3Py;dB@1svZ!Sm2m^h06~+->{(5V#GR*gvcrJW~~%l!FBEkRC(i=N8#zip4>Huibox_v%;lt+UaON1=jkfe!SsxoqVRu1-Zo;KgsPhE z)O#;ZM>B0fQe63C0($~t$Nig=v6uxvyc@fq?Xzd=7!lD@h3rlU>ha61Z;xO;33 z=8P0VnAJBh*yn!X?Ov8iC5M7r$u0XDZ?;9BsRLa<6U}Nj&7%(D{yz@JFF#C{Sz#gX zN(uQ^bxfY?g*!5!B-j}Qsx;s0V9|MEZp7c-8;A;)+hL)X>;uK^)**G#A7wKFlSQdK z#hb{0b%=rc%XPSlf?gBFc=}{RNJ1R_h!SaNWg&M1(B;@*(eBCP5<0=WTuj@v zaus1a;RT_ZKkoL!&v|ixw=}#iuZ&FDLW8%GyrZ+;$GYOaRb-8CHP%Yko{3fU$CIm! zZ$K!x8_VL@4&<6em%*zUY@9uL%{-Re_7_%xpPqKUiQx=No)8S65>%+=h!z+&0Lfj4 zA5zZt{{oocglI%x^82bYeA+&34z7Ek_th8rv`@20JwnZHUeu^1!u&-H(6>3;JI%9s zmBme*W>MCnSSB>2!J841cRVzskkkSWSB$0SfA3}$z)9;W+dsrHvU(~gi4x^lvl?J+ zTVGp48#Gt=t(*oOo#TBeOA;-uu-@$Q{KXakjq%xxKbimNE_3jtLuN3Kf8F}@YT+G# zVyd}_Q0mfTOx@2O#x%w{;e+aN;CtBz6o-{XQHA2@nuc^IVt3_Vu(*jHF!UJOwsRaD5K(h z{j+?YPM4b>MN};4r&5uMW{uhRF;=8zwmoVDh0Up?!zPcS|JEd*AtEvc_ zTA+M55m@ShOzS7pNA#4+Pb1C9uO*1wkNR`fOHSRIePH>eT4JK<>G$5<9))J8SG*ji z0o0fnhY~i^XP@>2_;h<*2Z{ELR^Yg%Oymh_kOJ)wp1cYg+eOPDLGj(l*SQ#PuYvl& zBb;cy=MDj8(3!GQXK6y^tg40C)-Ta#1 zJ6J}UyltX82FPq!S+1W<-SoQgyg+8D6}N-aUj-8-MZ$9n)cC(RSq?!gqZf%95kQkg z(uNl9=_C|ZsiOjOH?Nl&n&SilZbwz7!Mw1wJa|Bp3gQPc?|XuKiZCafu}NBv-8Qb9 zfWrE&Z9RQFcpWm!o09C|>wu$>aLgVCQqiyU9E6d!rI$U;sk|JYlE}8naH?Dn+%EWa zd@xjJEt(j^#N)Hinci?TA{488BsSxFL3`+yA?8l+adS1m@w_Iyp`nO(t``+|a|SK# z4kDy2c%%%gG!W)|YSH*hxYUdhInx%Rt5vZ2N#66QqRa7B(5oYd3$zH@@Y>VbL#$Q# zClpv6X|H!*`Ax-Q)fALed*z9iE$Vk zdD|1ihG`>X^lX3;TKqIdOxNgoyIq~DPo@)iduqCPF`D{a@^o#Gc1guDOf-ChLgXev ziyXD*9|K>Dy?ohrytzz%W`;J~CV)B|0|Y~XpF-LhzNC#yt^D|kq-`mL5_06EpC_Nn zs$WS@Akp!J!Zr;?AGXqv!fL=up5yIHkVvP7=sC>ZFX~Y|z#75=)#oGT$3^TA1q9C!hXM?Vf9GLGJ=716(>9%~H+$ zggAmrIV?w30bZNtH~tXzr+oMj&3j_qPpvtj@IO3*bWD#fYf1a-{{dEM+RAtQn+5zY zkX3mbXGw$U(C>0^a&l%FEyfRyFNZhY!!XFkB)eV&!Hm2qfJF@#hxhadHKt|O6a#8J zG@7d|PA=bwjq=5ihA`jK<>kV%1`Ljj3gjZvVmHzV$|?ILfTEIYcAB6+m!8VL3t3)^ zR>xQNa=3hTRJ8mo#$oo4E!1^YOFE`tkB|TgsMi3I-3sq{s3+Tc(Qc{Xi6i;L^MH`1 zsHNL&OqCcu>68Zd1JmW-pO>`vO88ux+|J*lCvQ=dTV9){iFxyuUq9-tyc!+;d~Yp> ztk-!jW8_7p92ZwkdH?(2G^#$bOC$5A%EPb&AuoriL(ZWz@t`nL&Oqj3QPIceSy?j9Xa{_ zNx70ZODvPe;-IC_^F6#2wIsq&I`HiLoZlR^i$8YBVy*t<~`4OCeXj zm0%Wa#-UU(TA(c={Z*cE$(m^IMMplAAk57qamc8B+bkckr9BpVy4q9F0|x`!=|ZrY z`&JivmOLK7hG)6wNdmXd_=m>JLNQD#6j_q z% ztbiS1ouJX~SD;Xa)%4Lj9F>^DbZx1{@lj9)Kh1R z9-cgq!b^z~Mj|~VCJbj80>U~y0h_i&+KVnAn{@zi0vDTwr|}qC-NTEw&v-AuH+lQU zUQpHm_$(XwWQh**vV`AV)$e3mqY6tDL+dfaKb^^58;%D9!i^Klw!NuvMe^0F);$0q zqSUhYkJ`qZ{Os6rdq}!0FL7$xRR3$|85>9Y>BMLDD7*EQ`1UX{zuO(=LT2n2?gM*( z2LC+vx`N{@7G)gjWWF;1uRnC%M&k7?yzUPu5^3)@?J1K2hu>bzAM$cS`&Re1tLQ_S za3gUq&}LhJFFw0oITo-9Bc2g7@=-sCyUE%`Yfo_JjsN1`54hcO_d(2qao_m+Cwo9J zo^e#Jpx2zRys(P5Gv#vtqQT?WhZufk^#l4SihR>v4uCYD1B`Inf3@TXDMZhx@LF3H zV*w@_1c-obGJx=X>IqWF-&DTRK3QU4OD4q0D%IrNCLIsE=zivQ+_-4!NLe2fT3!ypxnUsKDLdek&tSWz8m&beGs)(iYe_3#8re@78n-Sp(3e>5qD^ z8%`ZA-j#6Sb(_sn`>0eWi>P!CpE~mm0*AMvGqIxNvluitt_wdr-)Sc_fX}s-rH4+L zxxP&>-wb$|7%B>0rAk!T^@;#jmr;Q9?wge;1ZGMj=Vmxdm30Ilg7yr*1S)WaQCaMxa++o@R(?mTPeV-^uK1%5x{*$Ct&6K-@}n|jSG7~J0F6b# zC{)BQS)`&g+}X_r5LN7&wvdMV(cIBA;tFf&a_+XfA75AR7R%$+`pn}1i?JsGvMqA0 z85|9{5!@2qXoDPUWznlwD`YgZd(9EuW-8D$kWYyRLb{wub09ahYaM&=DpE;s zDyUO;p!k|6C(iE?!O2k& zQu*j>8cs^;xSvv1u2bV6j6u}PWt-o|;B~ME^?ML+-oE1~aKL>qgR-<3EbN36y@<^@ zlrvfexDx2HyZYEEk}!3h@-2j*Px}QIYJ{)R85-m}S6aiDsKs04T&Fl$z%OZhdd<FKZTzI0m zb_-tuDPQrn`oH+3hWNMratn!8X4HQ z`I?lOJRz~?Lw2q=nJDIF`v?5TS8rvnA6QwoRo45?q10incgK)Nis!BXxqo7Wvz2T1 zyU!z)3JPY(&~P)f$mN0Fs%c2vZwuMnl+!g?*Qnw ztRR5U%Au1??aEG7c+URMfLMVsvqC&o@l=Y8wOW~{!EgSdS(d>DfrG#Y2EXkPg>LkB zKzOZ4u+=lGxwrt56-nqX(_lBK=9|u(`!iZ2N4drpLyDQl$!&9x>TSPNupRWCYwC6L z_u_~5j|*K6k?PjAmuQaXCz{;Ur>sU*Vusr@I196v$$WBZ0OI4};gPjnq=dLDrGmn+ zNi2w?qwO2i^W7>;we+LzuY*Yn^J?4q8r9jg4^-31#mE>CIO zgL_IHrkw2S>p!>QGZ4)s>Sn)S_FC$`-V1F$^6hDD+q_pSuadmJm=`|PCcVPgKmO*& z$$_pBI0YHU{x5c~lUMGS7xMw5s~h26xIrzj*Ey}AquIrlSGZ=9TPJQ1DhVelJ`C=( z(ub`3t@8=Fv+uKB^G87P72Trht2WIzqVKde6u?s@YX(p)gM^CBVX8Inbe}86;cbKR z2p{emE}z@Y^&AWRm)z?!<0%zrvO1!DAbIcO`p&9-4iLdU9< z_dZuQI=kN>>`6qWGd8z2a_*hEa_{X9Ou}mG)qJaXcu3 z_l`-Rbp8_#m+KR3I=&HPm0`%xr1RHPU^5&7pfeKX;#m1+4K3WZUoyE7E^HP%=*&8W z+1<4hm;SMRb)%oOdVf($7Ri0CTj7xlh_1p=z*`pVsgao>cUZofD|;`T9WDp9&iKL+v!U+w#}@wecTH5nh)E^uPu{r@@f{H|KF(oT;zON&3rwP; zKmsJ2eAWT91{}`EG_MR=OytlOPD%Ui$Nu`LS{X;DBA(HZS&8YFDx0;%=qligy0NS3 z^0&S6)eRB*eELRwyXpsNZn8oVK#Y9Ruu=zHS1@6GVqppj0w!g;?$4cX2`O{8#7oIL zdI&&BBI5%a!{icEWJsn8yk$^B^t6D_yO}4|taOljDp~w0=`6swW!%i336}@z>L4s} za8aTRP`6YCdOLG`sKTrhFCqwPR~F-^t;1QXjeas>=nr`do0Ht_eXYv7vpQO8lRBDT zd-rh?3~0AaTcMWWu8-`?iuqfIm7ZSz_mWtGR-jyL@0ynf*9r2D6W5xpZq(DRcz|3> zHuU!k)nad+=L*q87i2vtY;4doF}a1k;1INe9{`j?Vc3msOzAY8QCwH~#_+QODe9@2 z*zOYX@KGLfYZp*0PvBwqNcJ_Jee`wYnY3vARDZfK+lcghxl$<(Q#?{=!8Dntt1)?c z-orSJjYc7@`zg4mdkQv1_C3#dDlgq~k9{$Gn2C4;&H0CjOd_d;+hoF?R$5pFZ^;uW7Q3CUeZ5^jUX@jVtn_rfktYALN6Dw({;DMVCyT z0758^J+?9$_Md&5Bk9M9$Dc!CkG=K3K$5t4K6H2oZamgGF9zq~9PW^)L8*70M^NS> zb{#B?R^>pSM{+oVQqYr-RMgF;lLr*-{e|>fVn}|7TA*nMJmfD=;(gUBEPp$(qb1jZ zaHk3YCAt+rc;n7(aiuemB!mRq;syXEcV4((Y=Q^KzX7g=wSiQ^TVwA*7>{Vmi1&2f z6K&}uWp_|H_Awlt_3TW@PtuH%z2B63PrxHZ;En&&O%g;qCFsKvf_wtUi?##Kx8W=8 zLL0pe`@9ytpBhn4zn3Qq1C=&nA+|oOs`;+d#Lm=7mfvU3xr0d-wl@^hgC2ZNBpl-9 zfW;CyksVC81OV|w*JxkzcBh_g*@cViuP>p0I!gtkK$e%k;Gk~?g{V1sL{W0l*jvcz z**Puuh1VMTQLa8@_^-W)>HR?X72R$50|BaMcPNU*(YxKeceMRZal=COfS>A!smMa8 zs9%$H`9+WqUPi@>xvb z7h73CV$R0W1!aK~N!}z0(F?_jTeRxr>x2l_sMF8{M~`|+f_b#qRYLsh{v1{v0|uuC zXH8V{o10W3CWAVPbxC2?`fX4?DT>agdHreGO_rONRUKD2W5XiH2YX2ZeBzEPLeS~e zuWxi%RR7HatWp8dsM@r`a(gl+nV^Rl&i+f}p){k0kAnkyw6AOCbi0x|CU7)DW1Ud~ zhg~~)SHIZ)=uei&FUBzTbW$)cUKN*roeuF(0SU+CUPv@ZCN?j};xXrmsIHE}4Gk@f3O5A~2;;5z7HDJIe z1346vIRU-lx7UGI>5or^=%vZb2XI^b3;jWm*DV!$dQ-S2jtg$Ys|jETOjy6tlM}YP z1b%zPamZGQ9-=%_F85l8kc=~h7E;f;(KA~2MYpW4xwKf!c(wL)-Zh7udklDd6j=a^ zDkGnL|IZ5=q?7ztB|d(N?P^4>9-jzy9rvDw+W)b@ddi0fb(IaSR5be`b$h~&XeUZ$ z9p~aZ!m8iskQIJs0@R_&ue@3$%nz@DK@@yZ?@?%qg;SeTDQfatnwyq&aL&S*dE>;l zROrgO)oANZ%bpJakbn+^$B>BqzhpmyK_6s_`s>g(htm>41RH~C zTwd3HHwS{zo4GdYPL-ObXCtFa2);pn_pxofVH_C_`Z^(VaOWJQkjW)abj5b?bz{i7 zcNsiTVPCk4uCf&4K^@z2OVYy zf{)t@J~!66GX*WcY^vb>tA>>rHo{R!d|nq5e>1 z`7v}x}q+Kogk`u>KF>p$ZNk?^(I0UHmX*#>;>&r4n4 zW4s65QB+($y4mk#KILP*rFu#_B>8S}GiHsAv=kVlZUgEv1gfIFEvM@dPDuOFGMUAM zGdD4)W&l@uyv;;4qvev5fJQl%cBDctpejVRbIc0#IasTsGu7}HLQ>9cafyHTCY6Vy z24DiX%7iQ?TfVXEFtR3~D@w_P;sUKQ2cXOGu|i@x8H0dqNI@Fz&_C9bDfbK#kY;)E zfe@sZM#+qmE=i@nsImPn;09i`Ht4w{WCtL`bZIekJ6~fTj9AkqJi@f4E@p)eBEUu-9h*~-61B!5!sc`B82gb?JOo1B)iV{olkf2J;cDGO` zJXRRJ!6QE=p4Aq5NI*K-gq#wGl~w~`s+OWd$)+kv%l>QEeVB}aS__6 zx8!osR+#GtCGPYqUPt-(J%MD_od>M{vOWv72?B8ECYH#* zej;Rzcgb!?(`{LZ_s7~z;m| zE_=r!z-g17cI5>`l;WL@gapvk6@&5z$#d7L0MC5z>ZpGSRvZs`Zv-hOiG4XcEFH7~ z=wZXB>YGD*F`1VatDQ%iy}W)Xr@mj)H;7?UV4dZd78J8@>t}JU(RVg8HhGz!_sKV|-N}mY)wX{vs#o zgM>8a0oCwbdO!s%%@1ErV48?T;1@Fy&&UB>25Vaoxx?>D;X>6p+4Cs(_DL?i1Wj=M z`q@D1ZP#fKgyD$0xx&INQ9kdqSP=Y3ODuvz%zuLsyVs7C-Y3uvzz^pURZNN;L$kPdA$#X+%gHITE^J_BE8+i=g7%7*(6oZ}sm;#_E7LsYxM4<^b0LI7G5+tkvlQ^#hkVU_L_zcM8D$4KC zS>!kmQ@IXxbkX8oz{+u+ss#p207i;1^+A8UW&r5TBhH9r6xLFiw*6T~chT2{)POBp zOpNR2K05`m#W${WwH4svm5H)W-$Yik-OXpcrX5B_SxKkzA z8nHSxuCc}8vbmMEI)Qe@eYW|2yonENyUH3yn|3p0Yo5mw&q#lMxm z((-ZmCmseqp*n(J$*lDf9C2TyI^G$thGrHqLDXpny-*J*0I3trNew$~P^|lmFGsxN zWB>p?HYW{p_<&GGljMUQZ409#IWJf;Ur^NaF2L3J_ULT~W18*yya)T1<)DzPmA$_CW>-Bevf}w#+3PQ9iUjanL}l?u zPyw*|MIcz5FHFP;4)ZKwd^)Q9CZB|PG0Rvj5dandzDsj|S+~4DB8WL@sxbIU_a={rXN+U zC(~+6h7jfC3=8oZoDp#PPM5jg*i*BLR<^o6WkiyYcASd^@m}1SB?Jr>2TJ*3wIJ@H zO7?HgkdDZGyN3hoqtIPOzkU`NUmWIv{&@*Vv@c&iRUUdwC$-7bdeS%La%LUOzOD2i zOvZ!lIg{q$Z*sXXKeF6(VUhML7_7@{SD#!ysq7Q#PwVtHQ#f7hcu6^ux#sK1T4&4f z!JjodD#!S@f*@aKid99+)iz?>!Z)pqj+LFfrMk^k3h|tp{-0Jaj4r*7#DDPknQ1E<)lYJApoR+gIKHVQAs-M@3?W0nUfuO69jw9g(D|`clP4?`Ugs#wT5t3eg>KM#?C}7*E6F}mkoG(;AIv`voq^O7MFt! zdZ9SZpO}~uc8O^dneVVlg_G$ z*N6=>ImFje(`Jaa+y#aPit18$>UUMqP^r6#L`H-sgIeR+!O-}CO;sZH_joe$Bqpj^ z&f2nBlFvP%hsoz|iPXgJaY3R(9(};?u;b8CIo0^%j0>FAaTNHt7y1WGhXp`Ir%c1J zBii6FrxZ^0Kdl}RTw+H%6bBs46+un6doM}ddOBW}D?BxE{r15Q42Jjmb61Y97%B0u zO#D7i)(-RmD3w?iH@k3Oy%SJZMVwuug6JmL3_9RAujjuCv8`kue^rVc__>Z!uDGF! zzH8nC9XHBYrrsx9{_cQy7?aOMC&46I>o`lJjc?Kvv`ig4N6`@^Shv%u zDNK>~pC|WEnf`KDGDcXvh=nIP_Qzamf<6Yo#e5XKW6ufy-eg9u%?FrEI2H}Ry@a@F zk4Dp60Ii-Q5`BR>F(Il-5}iv(%mHuQqs~OAl+U6we{dR_ulh+KfE8eF6NrG z=6vQH;~9g9B&sQ8j4$YZqNKF{!X>&&HfcSVU55F6H+5(#tLPI0-X4;@)Aetc8)^o= z!C88Ji57c(0PobI>1vV=lLaJ_Uxr6{!+XW9+OA3cF5A6|^qkjO!~w%0d5YOTD;D#! zkwLQ$+vF>cL7N3xU)qc>e@{Y0NC4(<@PA5=sIj<6r?OY?@S*b)Qbr9gDP&0_WnZj& zt)#pnGZ+xd-HQ9c8+e~2R?;L&_W8;khdeDz{cjc`lj*OBO1>?94hoFN#@{DZO|o_q78y`+ zd0?POl0r=6IGGPyw!-)~uG?Be1(2W6KZ^K4d=Rw#{5SI;)>o%8czY0tGq0Q@C? zDKIFGbb|})9PQ=Y+W!38_BkSe`M$|9CZy=tkQa@F&K~v=XuTUf2;zU`S^oa>?Xliz zZH>xr?wuYx@IKg+(nSjaR8HncHYnY2H(-7Kp3c+*E-3EnL_o&`B75NVVYX6+WqBXz zBaemUM?bmct#1u<>(Xc%>|ej83F(|xOh7!};u$sXMCyHOn`L8NP~V(X&1GXHC1xAz z0$T9PC0ICjF9BDno3L$;1GjX6HY>F#Z$!d<>2xJxBIL}^F zxU?dgrp3D18`&Lq%M`t95KO`l5OinTpGGaiSWZ9wG(7MWBTa~8xpP^1N%LMHoABW- zKy;MlwOro&P3t_R@=q@gDDly9 zqhn?c-~{Nmox>a`oF*JvZk(3^gNupJYz+&M-1NWYC;A6i?DGE?EN(BQxjEhtuvwmi zQ#@lpy)tF0t4h|5)pfb$->1_3YA_i)L#FYoP;h7nIW2xyStGmvI z7uz{S1<&*JfQwV|jsb|#9{|0&c0HT3!vN$Po(M3yr*Yd7G*7o!^7x;TP&%j?8abIB z)By;r=-mwmL>_arHR5SKOylD~)DA!x<+*N?od4!#{IFI#=~n<4oCrY*1133Ym zYsU$>mIj?x_7L>~3av^Lv~`&zDqDKvfvw_ijz_5iF5q!t*ufXh_yOn7pzs$PTnN4H zx%*>lm9G3OTHKd|C7=LcTK-7C{)hYB2^^B~?6hZm-<<(}=+gU8Q!m+s7NXl8ycBNN z;C4=rovPS&R{#`7iH!*qud6Ez;V~L|y|6reZR~&2hicdAcAh38b@4M&{|FGBSW;WT zGth`nW^S{Q7Py>ar%wbI+N|)hPg`l;3hqe*oAyxZ^LHBY?)z}#V_LIZ{JrV79X4zL zQr_nZAmh`^Y-b;-jBoanT1|IwC;e_)oo2P8HY5pZ{5*IUnv(6PZbEL?=XG)lor0EY z>?}(gU9|xt%ut{qHn-nTNdfI5pH18r)}h;u*40>J&PAG|@_~uQKbO83e25~dFhV(F zd^Q`fKn5(_yLay{uKbomA!3~M}g=lGLgUa;AC>LHE)^m$YlSGPf;1(P^FTQ7;j&MP)=DR0+O(b*vu<*)5;1 zk(td2{mV=Wau*2YBMzG`rt=>)V;YVIq$-uMkm$0pj054-zG*mcM zv|2t_1?P}4u@+YLTX<}NqF!H9Zib*idlB51_fs`m(^ZZ!7j_#1e*7fvG_&H|_Er%n z8;lT%1e9ap>&M_ZfRE!}#8pwZOR{mb!Q}DfP?-HkZ_UV0ySp71-Dn{9=q`##kU)AIJFp&zga5^4#`*r4Nd*kKDN=A=a) zd>Qj@me?Y)R9u_>W5c8K{dt6r@MpZ?N3RA}0bPJ4!1<{c2=U-~IQF7#htjzt3UN!x zJ>pM}zDnhpCt4rwyw>F2kgIUQp8=RbKFBBkgie7>P_G}19!`DKn?;1jb*?iA5V{h; zsr#G)-oHzfSAO&aYzcpfZRg( zR!U1hW;qB>jU(+g$)GJ21xnyl*8{vlR9e22D5Of(0U86b%2k?}27`%kQpSwJ-=#MX zzR#9^2=LU! zg@6QaBsbc;&`;5Whq&(23ZwkLgdG5W`(y`@5oW!JxBC6mgibWdr+a$UGKDh&?QB+e zzaxLKf}fM}BYJ(=&QkdPVAf`ce6F^k{pJL|JBOM;f4xN`e&x)OLCgfxT#a5U%FaHZ z4Z~c(dB!u_NsnswD_hNv;)RB6|;}UQkQfWG6D3s*ZIeyg3)S;*F`O0hRA9%Qx7MpLQm$;T@zzE z{6uDA3X+_Rf=3SG^bBmirDJ8@B=ovV#I{S4F;u(SV+9XyPk^?7Z7Sv6y}G*kYxp zHw|A-@W+DgJ==<=IJ2$O*(-mvM%+BBCLG6t`xx||(9d<^+r)|9$_gVwy`r)Rp ziR`K^xl3UAihNF1bHG-qDQNrn7Wdoa72`z0E{4+Q{e@;a8(~j;=)A-#9dbaJHlXZC zFYC1(k)$NDevxioaAwAs5sBXAewZYV;A>kEpP+xbK7Kb>^QQ&xfwLL&Hh|5Ub>sffzb3@ z9+aw-`FvSiUuRpT`L#mT_V`dJN6(Jw!5wD1w-j*i&?&aJty+TX1=GP}dSn0*FnB_* zf-%d%W7KmFm<=8_0qSPCWfj*k`jy6|9Qbg6E#wd4$u&ScZi#07h7vk0>y04`kch`w zwoNLjN)z^^Krut@QcpadoF-qBxi!{QksP9hEpZ75hD**szd#14l*dauKkl@y1z*zu z{zXr4aTx`94Zg->UOVm~7&i>3aE1T`_5RO(7ouD(&U@s53qWXRU-VLER7|u#F0Czq zrsm>ahJ6tCN)xJ&Vv#|XPhkP>>k@dHAVd+}j*2k7k;Y{wY(O?hoS@WM+t#V&$xDsS z9N8csZHmle-ClvQCS7v&`Mnr83&;+ChuL#iGyA0!mtG++u(zx^Zd)0;e!@gkEA%-> z$zo(3fW1mCa2*-pz9@3OTsbO+F2arD!(_o7KvwrhdVEu?S&P-nCA-X9_jKgbTzES_ z;MmIwcpW)4N>kDT-Y0-q2!**7!DG~HXss{Q_CGYUgplOJ0n0V2ej3$Y>s)b#vDQyv zSZ`d6du*y!7I6><<-57UcG0zQOK6e85>xle&M{*@YbrR$6>1MXe(WF}Z6?SHKNBG> zSD_$rU%U4r^wy!hMo2+A0Mc&8Nxqll5b7)iXo{b0m|I ziE!OUzddfbT0`na(_V2R?qkBmpk{MP_un_TkEr zk&#iTgOEE2fPy1+A8pW0lCxj#De!)*;aZof}1^z;U!;&~e^s;jl>vQd!oWrjUe4eF~@H%_1 z8l5G1^14z~PR)lgXNw1Y!RnO4${sl!x;lC@RXL^MykF_G)7_tV*{M-0y&I40F74KAHE}lyHs(IPgE^i1*l?mec30QKXMNu~GE`9Y( zlnh5d*Y^*LYFVWXM=Rve**&VxS8t^p!V^YA(P@p;022S(9djho;cxolS(EWxemYGF z0N2dx=85uvzln8q2awhe>cB+92{?NeTZ~GFTPz|t)=C89NU%^a*UwiXEU~V3cyeHX z0e~}ZYPcmunMZ~LUBNgg_MTI>7>m@mju#yP5k2*)AeYcaz$ucGE`?XwmO>`&2|#~* znhnZ~hQoi>iNm0Uh!6?|9=Na91HW5dH#3@1PivK*+XO^SX)4bgg$OvHZ-cPEYdbr; zN)kqp@P^ZP$zZO%%x0$KoZ^z>{20*B;uA;>h~k|1&(6HVHEvZrJi@-Hz=&g3dh>M2rXmJn3R{j5-3G^Yt%3%wpD6kf{e7h#RBIzZpebaBMLi zI<&#&yz}A=81=ee@L1?9ep^QnV$6Wq0i_Bhkx_3QB3PuB*1d62eUIyR%D|Q5qwO;l zQ*Kxo7dHW#i)IGUVRh3E>)HBU64oI#MmOovz?ezAhFyDC)xUH{Ghnl*p+3EL;k}sv$Tri+E zmL)Mj>F%3a(lJmvP>&?F{xa@BO;kN+rT@i5m-M`extf0LkLm6@GH6SNeJ7V;JZhtr z)l%2xF)O!_C))&RP3GE4>;+FMc|A$1jYqFN7Tyi{nkW@8U+khp*Vj+)zS2%;1!&pP z2WceDL9?~{f%k`+6}E>*Cp_WSuKyxZUn>8A0k7x2s8 z1t@%_4sK|sVKyK1y@#VVZE<={jqXalrNX!x`}ig_pG6K(9|346&e|G)?PCoq!_i*@ zTaQkT{ z%3K-AXT>hj?Y7yU3pnozTjK=4a%-8iEr0a{ru-X;IoFPX(9^hb<4~0%?Cn8mV6KOv z57rY>W#%Hi9MUcxEpvde)(9Makv`HHMQ^AOlyInb$7GWn4!Go~Zdf}&&-R`7!S?Ns zdWf|bKU#Yq%jzf@BQ6AH+f-a#+7n)y3^P+L1s6)-;Y2wF9qr9skYQs0P9)==^?=Iu zQs*+Zpve5G>;k!qZsI+-h&6F%t_%h}EQ(uP&*MR4Hv^VD6=Y!abHwO5YmZ5i>Mi}L z9D7=#^13n8FFdymi3&(mEnK(7XTT8{0y2&--45)NkSf+taOiwhb~RQOH}6+=rxv3U zFNd!^C@bO(XY|PS(%uA9J&*Y$9o{a01|KIkBtB)SmfUV~py0v9xZKksu=FC^+4e8D znHl(Yj{|9qh7YEd5`rCFRQK1g95o0xD9-F3sGpBF}AY+kwbDUEHc6nlyI<|VPz z@-@EWKHZ6sD-#Vk?ft_B8Z^Q0bWjWogJDp9@Tru_Nq`pJExFCeZ06iy`-d)oUu3i4 z)9A=l@;3E-6WBivNN);Ezr>|hzyNa1+cQRCFaV-^-!!FOB{LPiMuX`9 z%vuI@W_*V5m1vXW4JAGvcL5w(K(4JFU{XakW;{nzMKk!ChScG8gCg8Q7`)BzS%!us zZPvabL!`=$dbOw%iqAAjZE=YSK)z3CwJ-qZ@~(sZ0R8X1rc3?s47D4<*rjzgp*wgI z+wSE>Juch<0X_RE74F5z6~oB^gNKE;fNkQ>*w@}a6l2$7@y$^+CE6^WV9TVVGT%ac zl&lLsc<7`W!o`O42+TdU{>v=I`9d3zjKko0W6mWd^myQUk53NV<_ z5;9(=4nn3rKdmgqm3c?uP>t$6(w$?d9^P3}LOC?gi9~b9NrIk@s*yLh#1fCxn?{u! z+DdlouXBO!MP~Rt_P6bdG#h=0!U-4Ab+L9xgIcWtAH4gk5xoAPFGtqBax?rMJE14& z=z{h+kLo3HS=V||H|*q6j^$eazF-k2m2?rBqRmoLPPZ_` z>XPw2M{Wu@sU3s4X*$7p5F73S1n%_VRbQp3)l_9W)ustnTzWoBuTJ z=83)nXRZ!-FL|`=`=B9k=qxmNm9LIFj+_3FLvDp2xaNB|^<9fK`Wk)4H?6B}&Bp*1 zd5OH%5u8d_=HA01{f--rMj@9evkk$6zpVya6-7lQqXm5tBd zKvH;LfFvcrl|UoTVz>W-aS#etODhL;#Ue8&3_uKM3(rg99dCJejAujz;gT#qn;6X{ zA{_r?v8aN zy|=nY+7!EnvuZW-+Qi@MJSSXXoq%K=!A<1c+}j7pmNyMSeh?2@Z9lNe$FOA~+vmsV za@b?Q*c|IVuY(XKQKeW*S&ea~LGAlDfRRDj(dv(MEl>Q_&jN8lp#ErGsF~8?g)9Ma z=tm~1f&Yaa>BB}0PCMD>o18s zt8?{9GG45>@ptF3YyNdq5gc%qAl432be5qIfaM7CO!3ui0@~s;8qDhklkbF5~13_b!L7)TMJA&m`X#U&rP z5)x@s@!j%Tb{EV;N{lqm08f&G9TX>h7GS?-dA!Rz9ifT#W#g)N%SU=XECw`WWX^d` zIQD(*n-z~6s`ZU6ea+@V_lXt9ff+y7UGKc%Y}jE^|0$hSO-J&(6VT@8+(`P@3VrNa zdGcsFNgyKu*B=R{tguZeMH~aUc-MjZc(db!O~PzfT%YWw1m>bUZRP+gmYfoYWS+YX zX4yXwhaLm?;{;2@$BC3rW-|YQ0uO$A{STiGf;f{|HH;+9tWKnME5i(1q9jcd8$bt3 zkD`oT(U?k*F?(FWwAQC?i&Xun=elIqdZ8fl?s6dAhMIAu?&OqGXTcf5jVqG)ULH%jU-5qo8GKndE@b+g- zl;7~OO=k6T-p#&v3nZy{3)?Ay1O`o^L`}mr((Yfm&vXM7ozwd2f-bO_!M*`?hpS%( zsP~4{U0hws4U?W&Pqe6&>w}SfUJm20N(fVd`%PC2!aJCE>kj7B`Qtv88C)*d^wn0T z1bjlsx1Q0RdkX?&UzWZy-VXOOd(r+>qE1)FFFDE;>QROb)qnPdHr+o;q+49b#n||q z8J{~qKuTbn@F{CefCe$|SDt-M+{TuY?wraLI)T}r?^Eic(X8-}E4O=XOw}zH6tTZr zaW0>EtLq!lR@EVlNuv;u$$mJ#b7PM*h%o>NXx&mGr=?yM!w7Q6d-RU11l(YVMTcFK$cOwU zTbH`yqC&us0tNk$4#<$sSsTJ{Trmvxx!$U$CI7}U8Jd;^(qtV_s}vk;D@Kt`U1t4C zYV)nrx_Ue{K+mW@4au5M7wJ|Wnvol-#2b<|;NC^4dd7=}en+yR;`0kEUNQx}fBXAd z;i44DWIGKICYsm8;iE7f@@)i+)HqSqo5-f-&#FC_V5MUQ3 zE8-6Z4l$sI`~ZIhfo6eFHW_#Yd2;^xgC;Aa2Y*+=*%dm;2P@DPLUCVf5TVQYLk;-- z1n7h+t=azWR?2!H7dLng8{h`2&c48}ldmH$08|F!7`f z8p1U+L`*&~O9(TmW}T<#Su(F|ZePdh*qSR+Nh zyLIU`Wn1w?5O96*2;-52h@cXRTe{9E_>)-5J~G?KIgZbjAwZ?T7rj@}=YeX*pN(IJ zD_?&qzyvS-13A#emP2}|qO72|2Cg>EH|jaA>8o9zfb+coo>4DqJPTArO}|G2c4+E; zPB=m}Ih%51QW8YGjo`k26vUk3%`W@U*la^IHCH81Kj#jA0e;%G_}=IG{7dqBYkY_bJZyd+kfME+zBYBwIOS6j>2e9wX1j5a_-_MFG@6&LgNEOa^%@PyN z5xwBx*bSCw#?SVz$%g|P2*98}?M8e9N`Fu)pawG`S$9FH)|T=xp-ow)2z|;zi{vwH zU>ci&W+R?1M9+#j^gfMn1PcCM!P^-K`sq7dcEIh4ONE9c0gZ|j5Lz9WIxGmnU2s== zn0z3m=NYGee~sknk4{Cnx5CUqN4;lpN(egyi_NbaRju|DBY&tSOW z)3)(OTAP#OYFq5M;q6g>;*Bt4oOZWiX45IW%UMvH#Y5vHlj#tQM=avbsE}-qyP~@t zH!hg^Mv|&o#YT$3l;DcNeM{vi`-sQqF0i3rUDDf_!upXt{r5*is9O!R0kr*~MHtA7 z$hqN_w0lB-zZLSLz##n${ZaplZ{Du@lZXUp=C`p|w5KZkiV?Z|QeO6Uqm$mC5#Ki# zCzY&qM82MX;y-d}5cWS7xFvjyj5JA)ICVv@Tw>tw>_@+QgO+@U-(;i@2u^dI6#iaB zz_=ma4_(~&S2TpLJ>p#HZbxUj5MiKsyi;}4_4?j?uU$vTCj%S&o-1hxP%nl5xzBjR z<_9Q{G_ScuN0iY=ydPIxBTIUx2V5oJwkgsdPoo_46^RSUSySknl;63B*bz>R*bIn%A@e=WD=f_-i20j0$bgOfM*(Xd2O84B!A-nw4Ou%o zWxgO-3DP1AI7Abie=P67Wi1jE0h$Subsf?Y2mM_1F7hNT!Q-Q;WHp+1%|Uco5_aTD zgLr13GMbUd$ksjIVN>yp$_`WMjBIA`(C8)h+8ivw)A3^Er&Mv^7KG(zFtnfaU-x+b zJx74ah~48|Om+08IckUX7N4p}*8c z2s4gpVZ0qa{`}b$MHqZcvHK~d@&K-|D$%q>%|2r&RJ$e9{*ERA_EBPZo*+0}6ps5b ztiy#QKDEzJHu$O%Zfyrtkv%*-HcLkHPv*}PijS1iet%AZ+WOwyS-bfh-{d3gf5tG| zB||$HIqmtek=A{P8deMa$%!#8vsZ{3+iph$!@WT8(|PRhX?3^v@x8=ua{dIzcarZU zze{p%`BhTNy+BL8=PS~te7@3M73e5Y5Uax9Q{-1rw)C152sv;w6RN``N!g$Y{x>2+ zL;#{WUzq4%YH+?ZX!~!j=EH~OObmot2fWl~_>ygWP+@%q9j1hZ2_(C-^gLmG-|paNrKe3@u+;Pbz3) zkH|f&;s1CrkP;FP%5c-ZS5Y*}{jZ=BQV~XEpqWk7e7m1h|G`InXe01tk)}XOcq1n- zFTMS+V?0C;zq`@}2#miRlRv8#QIgQNO>IwUcvlJ3n5jUgT14J5vrrh|M0H`#I||KE+L^I-g|Ua!~Y*o zR8k&rdI4`S%H;pomoSHbC~8E}5Tg(kEB|{k{pt>kMA$3^ZD7D9n=>8$r60f_MMy=f zU!x&1f99!X{+C){xWKYe83O}uotNqPFLemyf~|THB?*WzMFlIe{uTGVz$ivMc~zau zT5El;@+x{DZ0wJz)^LjVQ;ce$>&f1q226@R#309%2e~wnqOjb6q_`R3V-sGrO%=)K z4}-!>nbCqL3cQ24NVuBop_b&JHg{CUPJ z1!(4Q$Qem9;r*X;hsZ~evnr0SD#I6?5m-o@!1-bzGhi}z>&C5C&o;K7E;99wY%Rv%eH0h`t#UjAZ6!JXCkFv9%|pb2z%*qBcr>lY5r=YTiYigtaGYGL4s z8Hxni)jPnOO*n5vVVIvDtPLpSqa7*l_|f~lMQDg!hxyQeEC=5Y`scPA=*yguPIFS3 z3uz~_E9!JZj&P~@^KKOaoDX=|>$SHes+3Ihl=XOqpp0dm#T%81SMB!EMQjgwyv?8qKL1{#mtTNxqy1G0aPplL?qjPWh~0%&#+?hBB)j;OoTMs|6yEy0XFB1F@ z4uVa3-xU`9+0aBNp$4TMru-8MkjF%3tr2^*n-W(80cKV%6xdCScv+rASi2=S%IQy> z9Q)U`%I{9Zh(R)-8RyP$iJ(<1MRN{huQf3Sks9nhBU=V)eXu^s=l&DjctbMs@b2zT zFze#$0V#NWc{E!Le3>Yg(AopQQ{Qls%M2SiEEih{Mf_bFZ0Q*TbtnWlr)C)zl`$PP z58Rx$#klgSkMq)pJZtt*^;0jt@}z5V0YXC4n%fBh&o z9j4*5?9|aWk9yV8?aTh25ZKKA(c2lYUr2Sf%g4(6kqGkE_=vY9 zbE!P%73_K<_D=C(XsY8X$Go1ZV+t(R8#{J8VnfA=Eg zl9;%h3AY+G%P zpSpS+A_T`;`7|&#`z>c1Ej26-lCmtjTr}JtU>IryNo8q~Mc~ zSHsP-TXWp!L+xiCJee7=KAX#JyXk(DL}hp&L>#+$=;_73l1w5B+s`~VbIWbg)GN|$ z*mWh0$7L#PV6es>VTW6)KNtE^t#*RBWe!=?wv$~SG~F!l(>c@xv*(Rgdud0)>cVXR zrp%RelZtfS!(G5ykMwM4SK(vc7#8@jw_)wssdEB_=@MM>WQu3InfeJ?LdsQ-<KdMLhKe74XG`NguA zRkqtFqdMnJ+N3H^l52Egbo!8r;^e7ry5cQ_hBFI77Ea&ovooiQ!}HNV5=Mq0t-8<> zE3c5LhW?>2jEYRS=|O5uCS2~8?`mozt-8mSN=5S$p;+u8B1o$UlLICCalu_L+>)j4 znr4K1DP*F>4V-P(yElmUPXmUlw@=`g+a5J)s=BQA#rcs(tys=A1rQEf`(L)hcuA*K z@+oXxy)&>qUw~mIp1#FQbl|i?rISrcUku|}k_1!9>@&>2TAamFZgur+(kumJ4-RUpa@_1$oX1J-D)DLQXty_L6{?t3L~^<* zVCwc#!|LXCBVoG6l|G+TvAkB5%$Fm=C1kQ~A0QHs8YYoF|MW#A|MKR1nyYEKP_}V7 z+ey14z4fd+p}8?@ZJ^qEedl#hC1dv8P6%^H>tp+D|C8!$=zVlzURYI-!Eh4Pc6?}O zQ$efLTt!^++8ss0QyDY8&J6x9dRsixx|@MX;!cw1sw)8k33}6Y4s|Pb*cy(h zoh=<3HrGQQNcy6hFPZFj-rjH;(7@j`BhQ|t=rQFRpb=Fghp6l(Sa;hgx6e1CQ!LCI zo~_L55e04I%%%qVDkf@_>7{=?yJs^##LiWKfBZqRG$v@zBi>RP#vXlnkD10syPYh zpK!U~V68LE)Af-TE;48}Lk(2OQCqqG=#Q7w94acvK~+r{xv0F0SvQZlLdUOsy0eIh zJ%wP@!7Smwll;Q$@zOA}nSmyY?Ahg2agPCSS?Z2iIEU(72d#1677SSbtQ2Y6=0~St z0nD}tx~lEYcPEu@n(bcha-#SiTbk^>R;r9wf1e7piJ)t=&G?N~!x#!NvcjO+JI36< zbLelD$940H93;>|<$q;SkM2Zr!LBvBeaz-nt(vj=wMm3iOEfH*PIl4bG7k^)O?rJ` zHK8NmT9zyIWJ#um(VDI!LpNqL#}h}IU;_Qs=lFQe>!Wpq#q3$0MYskU*7#|8@VQz! zug&caEz!&?l%mCr`)kFO@Azw4yzLBjxJBLLiBdj(@E%RJ6Sx5pXZHC#+-mZuH7OJ~rthhGu=8*S_Dj6_dqbD2gCrW^Fr=`fL!+1=Cw zZR;gP$U9t?FJY3M(nQuOxJENXJ+poJ+jhF#B3Y$qbEH!7Z1mi+jBW7Js<;4^RM&&|y8HSKuAT2gX74;2RG zv@d|+Gm$hX7R4tene!%!T4^scPFNp%_zSf;D)}JclQI`(1gn?~h_RWjlc3AY>-e{t=ZG4{?$kG~UIMMss>0-OF zmf*SH_a5e|J}z@3Kp?|8*O6L&6K7<)&3ZNrj}N8eE{*snyOrJ7yYGkX`y}pHbihP^ z-rj}G6AJ~%lh3ty;D)_z33yuS5jvn+0_W@Bbj$9{Ym4K7_x!mx09db71x__StL)a9!{@Uhe>NUuX}J-$&dtmnpeAiL<0%>yBjq08-DuA3YH zB*NMhLim5ME7{5dq_<1q&qt`~QT4(iS6TPW?uOtbD;m{tjC0V1^Z*e%?%&4+4 zjZP~ym^`(Xf(=6i7?9>O4#);l9m;+RGhv5RpODb5G@6Ah*$T7}r>y3{JbSuvG34sd zOst24Q!-Hqno?Wvm&Se1HgmHFN0_%eiQg z|1;4u6*6bucf7KyNfxO=CJecfPe7L}uMb2w)<5#^iQTZC6q2fs7b+7+lvp9zSAUK} z!V<9gPVO@Fk?d8=r46j(9v8)Mhu9!fgJv-R$mB@cZ**aWj#(CnpUH3RO}OCH<_->3)=4? zlM)nrakpOF10Ql?ilr!``S}S_*OK-28Q)j+CaRHVJT7poxH}&L49Nws{ZpD3ZL z@G7wim$r6!na#D$OarU^+*=}k$3%61Yu&JR45Ta#?@84^v9BETXk?o{-}z{QeDf8V zK(1Ij0pYDQh`(k?q$s+Q>h=R1E49vrvEdlqpOar?Y?M?z=oj*21 zU}9{}QiH|v+)oYTV9gnk433F`_NM|zi5cd|;TmE*MMKpUZ13rf<1iA0qk35#>i|iyVkpLRozC(dOdZQ}tRE;T;wR-DIr|mP3S-OESz;t75 z=BP>0X8kk#k$U+ZQ+z28?9h`j?=sg_OtunbxuZPu;m2{9j3uk?zAJF7aQ>~Qu7+&v zmBT5|-o%ce&4jmfL-LIPuvnYjLikKw4pd#%U7p5#FG%u()$;&3$Af{P+_ ze@)?H1xFzAe)d^mhT~dOH?$~-hG8*SAOUp4?bHs~02%pY}v3okO)DxrhjXEmml?s)xl=ed6t)4H&k2LOsfQ?5r zcxj$vws*eeK!fyR7hz~`Br3+SYOenKa`)h@UXjVvrd`wNGWt>RW2K$rjX_ElS-ZYi zG8r9P&#s)?wqvxE?blwrlvdwIW!;UN?=O~xYc~uv`tSitXnRqMw+IL6|?Tk2Y=+65L0vG zgmRC4Q>MX&mpwVnp`Yl{Xf$QoC$Pk1wc8KJRM8N+$;;f+&1yj%(8q79mk})&ngS{} zJpAaw$RBS8YVp)C&~@WF6_~gu9_>5Mc^zQzhJa@!%%2Au)?LomJr~XB-QftwB>pBo zImh0nj2>^8Y29){b`?gi?xcESdJ=>^t7vpTtbG7iZ;J!7wh0hYjqR)nQB>u=zT5L` zR4@So_yD?RCPdxuRY8k8RxLRTBocW6l1o#^DalRjD%hBN%6>fbuSeI{&W zMLjk{r&(TyM`7TK6SYZwb$$e=y{yZyC%kf_>I)S_E{Dm?YVnn`UI%rRCLcYC8^qw; zPR3;_V%^SL?D91!sq|c2@pAd&b}C7jY+uxcHaTZXSIBs4K(D_%hDTQ~QloX#!B-k_ zdCy;qBFDTM?BCY=>A?EwBH&=tTR)zHOplI!!W8_h@?|W?v!4;b&Y;dhVH~-WX?&|D zo0c>zd|9Wbd%ujrY5gOsSFP-a_BTAze4^tY9{pTKyJRl$EH=FEhu7^=Oco1|xpwGS%ZeAPx@x0Jwm|)0vseQA-x$H4A$0U5IoUutPj6(}pcfNOrUn?_@tWiN zhiM?3eFErtwajIq<22Y{Q8=9 zCKrp;+in{quN4ubgY)!v{rq>$$&~tw@db4*x@Z^5;nIL)Xc5RI4{0y>Nxw`vY4`?a zia-;FnA31~<}+67^)`2<0yW-1X=h?Jg!uCrv1c${`uMMdbnPJrNB2#qjgJSPtQOv1X%b8^1afE>B9>6- z5V!?31|gp&RPbU9r(&Qtx;5$>+MCFW#_zT+Rwq#()iiwyK;Ylf7OtTMRJ~d!o2tfR zPIa4IMo4cpO@S@M#kvt{EnmE4ww@NRTNyJpccriH^O`Lf^?P!Yn&=p}-=R@}khr2b9pIMe2P~F(QxP_2G2Cu zZA0A%o~k8$+U1}ZCyGmOKGL6Qv*uGW(QeaT1MIxwoJWb5EejQ) zRn^kvMTlmo3fd2l((fDU;}<&HjSS;)3M6bDn_(WW z4PIx+bEG&8c9E9*xazQ z?onSekXS)L_uMibsEDLgCYO^qJ3+js`v^Jq*nLL7C5EEIO`&_O4?4;@O!a}jHHc() zSZUkvJ#79_h_(C3*pOX?9=#yDvgsx6j8l!zVx-_2yYoVke#&$jxIJnHkm_1}5pO(w z8##`4|7^jS$?@vxmP)WFS=s=z>6s&w`)r!x$7T}_lMG!q7{Vt@OgJLLy7CiE~1I3<8GYD#qc#J zmD_!QMOctEd=fy^iv*d=%F2H$$Pr~K2)+-1y&Lc}Q>x5huA+#03EhaE^1AO!q8xBk z$!u3f<5dN3m=`U&65)?gbrlXuN@q92p!Yn)0)<6$;eI+uk zO^k^HHcbVjafU|h%s@%8L3)nD`fA&0EsJSHh_#LY&TWamT!T34+(rZ_#m#@dG~hw4 z)n+Sw_hhJ$h8kNN_bu1*pzzwCAmsO`k6{FUqrn+F{8o%yhs~;jT28FhMuA|;k{BbB|EMkmHN;Kz=CiSs%@KP|41kfr%~=i#(4!&V zlYa^O{CPPT7lOiRd%u~aqy>*4nH2p8$^-d9Y7PM&lcOX7j86t+_P^`{=f@Co0THPg z=Kn!Y2!uR<@yu0X8}AQ;@E=$Z0PMp;fPIYlqA+_|kp7>#{bK-lAtew?(qcOxnGF8R z155Q70I+Y@UgTHzf6x>7AXumCiK0?*?&Jgd@~}!nUZjly%{=5?W;KW|hpP?$zswZ~ zfcT5bGHi?5Nii!F%1T;hcQk?TkNd@xr#ju?7uV{~Ll3|nhGEa6)3J{~^m3rc8tfw{ z75_P{9;gW*doqCFLA6gK^I$CW@ISvQxa3E608NOY{gHpt|KC{&@1aMhybs1@rcnQ- zYap}X0PA8H6>+&ZqyKNgf&UnRdp>~zmg7KlTua#v<2n@j_Q~HQMIm zsxJ5D8xKyKiOEqa+)sYW2Oi|(VJtK(w0a59st9yX^(`(6j5{5F@e5&21GCE^HQ~Wl z#YI6t>?jGvW1uOdqL_8A69}gSRgeE4SziHF$C7r7y99^eK@;5F2~Lm%cMcMQyUW1| z2X`k(Ai*7iLvRc3?(S}{xp!v%x%0Am^=i^pr>aXn*;V!J8pRL{@~*E5mJ$ibWVap# zb*I`PJQeoi>eZe#o+mq>PrV+0ku#3u-K=zuWEQr3LTi0IqAS*Gq!i!11q&#^J7M+T;5;2PuyYwjle$%dsoW}{6gh`F;$6o z1_`zGHMsQp#q}?*8uxjgldS4VT>Id|o^;GyUA%nGU2K=rQieEIjo8}aYW3^wRC2mx za)v@4O7)vRJYro!ts7`IkbfUZrcz9;wGY3`a1|K%uBfxv&EXFx;|KSK3L4lUzFt^0 zLM;lz644e(+GyH6=uVd8y1GWg=YCeLK=Pav7$^HF$-Bzj@33k>5)9YjcQQG>-1jwM z~165!UG09?FY`y7$UXE>f ze7>;V97?O(7$|E=O07xT{|V-EBI=`lW&2^|DD-vJAOs_R=X(&z(J+uKTp&r44*7FIBE#roN&m- zF&CI{RNbF|)b(&~Nk3!>7r7&1oNaP}wsf19n%vXs2cFgMueo^43JN}?y*--C4|~91 z#l9eC{WMo=@pfrx$>*DKvTpQJ>*aZ?%jRskQPnVKcSM%Ag165fQfV`vu6IVM-l~27nJ(lxY?`<^ps_E9THIZz zTGKM~2sYY^P$z|7RZOdy6t4U7SewXp*s2xFpkz-vP}b;M1`5QZKqX*Od&U%b`hFUM zLCRKHryae_j3`Hr0yZ|N^>UU?d=nlsew?oyPD$9OL7BYi!6;_H;oG%#k3wu6(rZdC`5w&Yb5$7y@|-RZum z;9#-Q;7@IRu*e3gHQu?wHQ93-U}o3K>S?uuq*X$b;OuhfD^xEW|uEqi{R0 zNAQ6GReIkW?b2Eub!u2cI*%4T^ToTgJ;-ysT_#+YpL4Ej8m|r(8l=BW+Pt$@Brj64 zr3X%eV?Ttrd=p&AHB>1mJ?{Up{7hJA)#kZX+^|-7^&UdzY|uPrsfux}HFF z$78&q`HAWCenFP=^eLP{FVPzcjcB1>aour!yWc_!YnOEwK7|^sLA+1ff|*DCT|zVa zY2*pe8@I<;pz7pin{ILkv{JV%F_)93_9{%En`Bc7LVymQ_7M-JReZt}Q~l*+@7=Pp zGzCQl41C}UM1VtqMvjI;=6!Xu9OWxB#YnVSXVA*V(!lco>Q+kriQqq)i_nL7T$d#5 z9U}L2LpI;NDmWH6n|3LM0}OjDzP+p25N)$#gO<@M@0;e4SA8);;b_lAQNP1*bbKDI z{Y(ZjwQ6nVe_WPoHHwwz!yv#G8}hmw1m?*nNv-un(KR@l*M%dWp_yW`+?VKZC7H;^ zq2RJJrgRt6>Bm->Jk+y66f)WM95(xzpB1xc9`7(?q|o=}N>bG(kBj{lTYU|puLp?? zOs~iC2RyvT3H25>%T#i|*4D_Z`Ukt3sfp{=$OH4E)@ru2r9XjThbSaZHf{&aH!4#E zsfEjr)f_Qs{ZgIOWd16H-RiRepW{_AKuqSncz+NX(r8x=%|N%-9l`MVevM|J=2zQZ zZwysh-iHj;HZ2?hl0RGDI;BFZh_AL~W?P@Lnei-ze5GYEeklMfe?=(I>SN$_8}a+= zX6|NEk+Q^x3}KK-bFRXJ>(8bqDMV{nbrL?X)4#C*&o)h!7F`SUkQk)1P@krIeFE7e z?g*#nyN_InZ4@d$KiMa2I)Q|* z@{7!}Pe8<)rfau{#)usm*Vkz80+`^gp%2U zN-3P$2p9KihKv$fSmpwDFvp#Ub+XMKT${PWS(X!Z%a4q*vDArNMffu9{ zM;(hbYN(F9P1CzLZBm%%B~=L8UD257=~a z^sGpJtIsXR%!N1!qLc;|4iSvWJc`11C#2x?_ImfnJ9KdSY6MSe-(CW$$>Lpv|Gnu+ z+p-$1R0N6t!PBt2;9^4f)x+&H{cNpi0)$Zb4`GP#M_nl; z`-nGsOFZs~eZzX5aL?_mu(d@R!#wO`I>OxRQ6)5?14%q}jGFau`ux}w(7+|pHo*Ht z_40(x3I}M`?{c4eFiPN2G%e59Ed4JH(x!&;EynNI9jClf_^oS)^Mt)_WW2nLWvV}` z(3lMpuDu%quszOJ?@%0pL=Pise&37&``;I&E6hX$JQU{fN*cV4PuE1eF z6!%^6b)oOkLJ+&{)0@lIrOF?lN>pVLFm|l5aO~IyNNEIZ+}RjSIG%& z^jHiUNflBW@E7OGelCN7CxTAgY{B?lAfKe3Sf+_?3l&1-UVLZgqS*ir1QzT#gCnI+ z`1x;p=nYCDus%rapXw08_O8;>pz7;?_JkqhG6-mo}|&}9}|Pj zzMB&|dz1yN=y!AUt2M&zWA0fkx73&nBz6({$;ovhIK!bas4Z>#pQ#S+;L`RB2S_;(9QAh$0uEv|sNQt9UYRJ`tE%PkEmr&+>Ylz0SX-NT{_i(PIl&e57O*R|LXL!vJqVc*G=_s=O8k)rTcWMfFl{j=t^7Gt7rd26H28}jx zh888XDBaK~Sh$KN;8=v9y&{&zpW=Q5F7)@wVHs?o$c@1iJ%e(#w1^bbL0yScaeZ?{ z3B(R%;o&-)#SYmFy4#o$y~gTS1)UP0dV92vcY)+j(-P(ahYlp`t33xh1X->zj@!&g ziDTP+8sg?VhZm88*>H_0A!~5(*qgHm6oYx|uZPu0SG z7KCC#ww!f#zkwu{I(-lVF@*BZKRZ7 zmt?WU8}Wf*pJpa$^}GpOBl= z_4`{dr5zkER?hk_b*2r+m}jOkSr~MOttFo<{gzjIM?q3sISf>UxWdO%T4$R>xfS;7 zH?L9cz}C{ybT^B9o`P7%rMiTJRxUCGJrnk`y+RL1-6C3Jd7)GHBJa;O_IfBkq|fmF zV1sR9NHM?v9rlsNy}$Sig;-u0c;^&)9HW*Waf1IuCU)Up`G6FTG1%{WE*9 zLz}$s+Hd3t2hC4gLk4eVE6r4cum=iwtmkNH_Zc2*gOry$Ix3X4ba}gd#ljo=zdcT9td&ocL$pjk$I^sL?+vme;Xpaa9 z<=bz{;uXlndi02;fhj`W?$6)=UU)lFfoio}X+P1{@FI;e*`uZ|cxOdGacgth7Nby@ zP3TP$bC{uV{j6UL+&nby?>akdl?FIet{(0#_nG$rE3KZ{y$2tRvN@2Pp7|JX*7T)I z5Ny=Lhgi#*Z9Su|T4uvE+u~h7+g{IUIVWm(U}1=cX4`sRafK6;U{K^2S8a1%yV3|Z z!=DX}fjAL1ARctKo`-Q;Ydue?=^WcWcw@ZZLGFzpyUlFSdr$5%$;oTQg8-LaAr8gy zNBNVzwSZL!ufs+%E`z#e05*+WVO39bs}2SNo#a|)96hZ3i2Y)*g+z*B{I3v9^8VO- zhKKt=-g(pEbd?@*gGuR``qMZkvF|@g9 zCFV2O#nLI!9=Dt)Zj2=L5MtdIi6M4#Z`;wA9nH-(X;CrdKI)XfSKTIC^r%zNPq5C- z9+r=k&N-fKyw#Iit{YMMWsSJZBzgx>9tCk6>BNUv!`8dF7#{sxWL0iJk@ig=uyaO- zeFcHl+yvvHuv!5o>#~~aGixs*1pNt`S@5X8vzfdx%ILsAZK}Lcj(zuS=z@0F#(fTK%Fo63_ z-G+x0|CfznM}1<}n0O|gMX6f!0d=-^NP|tnYn=9S^B8MA&wiZ8bgKCF%jX@D6{WsR zq3{*>s;tvS`GAMg^iidU0F`6F5uJ|WG^UZbuUq<4FWBI?qaI}3g|f?aWhBwiVMPa1 zYqR)v(enwZO!}zI=n;-1Y0{E&Or$+@7Kppn|7D54#&U-U(Nn%t$F^pS$eeyf=kdvj z3|AchoD~=c#=N_-TppFL57u*CYR4?K66*)D*i9YOZcZp`PjP7F4G+h@4t^5_du`GC zV|M@OS@oMe13!8S1PWeu^Hhy2`X)KD}EChv|f1;-)~P$Es_as;!l|DA5f*Oya!j z-o`z8V2ig8Hy|SXg?3IAvtrArz8`9T2KMR(xdlPfZeVVX!1hHuqC~h)gTSCC> zrz-MRNnvC{zvk_f=I<2CST5G1l6Sj%U>{RgeH|t{ar%DI+4g*rhN-Oq&m`WOmaGf1lkbxQq>0k z87cZ6`$|UnEP@$ksM|v2ABlQ_167Rs3l8QdYb%-TRacBJl@wr*gJumpB_ z?Ple%jz+%LKbG;mS%QdUJcEwhut7b2bcL?_SNZP{q6xP^UM~r1rK9wdu^6Zq50wx` zzx$N@zJtkbl2Woc|X>bWn_zH<;_ zERY*$masc6CEs&Z(xvyBsqNU0rp+a#>nnaN+cVl!@)tFRe&3JJF+SeD@md_+va{?o zoFx6r3$r+R@6Urh7OOlpZtu%l_Ikdl=2xbY`6!W??Pgo0i+CzC>+$Az&a$nCH_cR@ zZj7pd1N@^{2U87rU1f$dmslLP6{hOn#z_}J!|!@L-wDO`DT!Rg5>J2V8`>{`A2`Aa ziP~xyzzp*3shtFvB-t_)(Tnx8!F<0VL=t<`MKm+gQ7P)x+Y2cL@@uCmd=ufAODP-B zPu@4}^K&7&A+Db@l%r$XUVxf*l*>%sS{<83-dyr&iHzi37Y#%-{6f^Pe8ViAMrcYI zGYm;AWC;laLkGWz>R{l{_FVB_;I^Kqqu6ENdolBUHpj@>y=G6$!ZlQc6n4`;W*fbk zirK|En30D8;LVG^X8$OZ91Oyg@;lSh*iF@Zx%VXm9PRYf8l?v?4N@5^m9Ezad`^Z% zhE^0GMV>Dr78;z@rJgwyB2;b=U}q=~r-m5dQE2+@oo3R69VRsOxcn-3eVhsm!){xI zh~aO`7h$C0-=h}Xn|J<%>0LR?ui891bcA_79(iyz+vD$|*X*TR2bx*EA}WH3B$=~0 zI#sE3yw~2H%}tX^VAN9Jw$`N8ME9>u1ux|`F-aHcf%on(olXoTh0;Pej6>jF0YmTMc&b|ELj zWwR6^M{NJcjB~eN3WO_XrtJqbvP691<9e=Z77vW@EM3J*Qgz%_pEia1ygK9{wDpe-C#%c$lES;F@&g-x0*pCui?IJR6lcbgU(c z@%ZuMQi3-%`0f!NaXpFv_1SBGL;D_bp8B zGYdNJ#rvN+F!xl#Oo1OCZY<#zTb~9uUEsDK+8cr);uxgohFxOQO09`-VPmoYhRKP- z(E#|CcwQ@Iz>$nK9D#-ht_Sk=p2Ic?oCaP$oLZ(CXTv7eafnC^A(w}7JK{EL97K(% z*G4PxB?bU$60|OakPYSpMu4@t*nIpARV_bmZ2o8 zKU>X)5wMgxmOpd#cZa+-Dm=ryIM23;KXl#ob-;`Kg%x@Bfb;7_^a^%8>Y;QlPSBm* zH?e-X&1ER{Qnkhd)?)rhNH1u#0;EI0C5$e9FL*dIt_!}Wh+@@;&+O;HuiW_QvpnM` z?Ye2q1x?8JT`C1o(rl#avPP5sAi@W&N|RUs^=dD|n6L18+!WA*{|+tqBCep2|M-a# z{e=NwBO!p5KcH+vRVBHf-wc-rJ2?6mlL4rt5;2Q`&nQiAH47EAI+m`Wuco>3_E5$f zO+M9HE7`|zvSh7A0FF?&bB$kNI8Wxnu*m?FVE*ue@svwfWkaR+{ui(~@q&Y(?DBS0 z7B%2fW>TYbqN|e|&dX+pb+}Wj(>^TS`>Ix%;ptg`ZLtE+nS;W%TF(G;(+_Sz2Kn!W0jFfdL|akXM6=Bf~cf8#zEFYK+oDCcjwnhfom$u8IzFbPvy%i|V1&8CVsm{40z`eMJlQb+pZ_ex#L|M0IWWFxb}|1}BzYR8u< zz6KT>#L6yR+-QQ%B)73jmux6`!E{f&|S=Pn-evQU7Or4}c;@uqDFST6Xx zcmKWzn!|8X1a6Ol=4C0Uk;|F?=eK1P{SX3WgdPsOOy0D${7>HKw(wWk(5vrv*jSJQ zNuZ}B|50n8S-#0ph`)YyHMmdVWb&%k#BaU*G#gDBfgUyg%2uaQQ z8SfL>DjE`>obNj)lan!1KZmaSuYbM&M{+j|A~$#suR=T`An`Y;SiBGm7%P&0e9+5h zwS{8=_3FA|bM1X)$n?@b|Gd5_0AHm{=GUtt%+=#*rTNPJ#63q_uF3T&LWMFEzK-gsg+1NkGqS864UKx zH#qXyFU<^0bb$6&PKI{to6pJC2rN>mx63e-8^d5=UqnrOVv;OCY>M5awaPDX?KI~-K z#J&$*L9!NYTUqotpviwwCy@-r0@3}amHzY4feT$M=eOiOzmpXnX@l|6+A}2rc^@Hg z`-mYiY6TRgrHbo&#GV}Ug6sd!B}fxJyOR|HRICeYBJ|`dI#Q;w&+V={**v}0$1%5) zv%X5ne~#A|7s3@TJEWBTj$U}pw=Rv{xqDMF`#-~h`jtMi zp@4)*1%tT*Nl2JZPh#(})Md9K?VT>ixFG6OQE&!$>C;?D9N4~rt`gWAP0pY`wY1-8{)qJ*1v`A zzwY2ylqXg+v#R(>n^;!F53_+&-@&si=yL)C^KBf3E;3M;-rWO)_;&jx8rfhv>r zgA=3`h4t?#OpgBNTZ&>VLr{pL)n)2QcFORzkK&=+pEO?D!YxgjpMJ>qUA4L z!lDC>*xR4`Qd@P9=cfuZLAEr2Os6iPJh@ZJA)w`d^Rzy6?D z0OU610y+`#e=7d!4sHhKUy}(?oIuH5pkXpVP5!5|{`DYQ4ivlsGke)MZQ8LKHfEti zXHi{J!GD7l%uAp}_#fT$${$V>@87vadvSol8FpJL-?&-SX)rLz%R%#gHD0yC>b>&` z6`DN|J}e^XKeHzj$58Iy^BQa?oUC-+9+}%3<4pai<6kPpzv;f zE^LQ~8~dA;gP98*9-F>L&uh|v%z-G++p$}Dhx@HV!wvJXHh_VA1|as}Dc@p^{MQK3 ze=#v0l$YuqCGJS~>DU~4Ua=j=NSBge)q^x5kx3rxLM2}3y(eouFgoHqI<{wbJR78# z4qqV zn{i#w?-WZUGDArgyaC@-kiev2e#$1!NCY!(yhq7$X&o_YpjSi+K82;2dZy`1l_4a3 zKER%2(HHV!O~*?@0_Yw|uf|#V93LQO4Q@xVnb+3K+Xnb1D?gxTgUCqjs{d{l8#p)oHF5pKC_(*7MjZAiDey z>vLk?FFB$00}V`Q$Hr?t=Q4XUhxMtUKq`ied)0+ZVR3XAjIx2@0?rrJRtBx>`PY`y ziHlFB!X6hgFdthUaPHh$bDNx|RDXn~^56uyY5LJSvwE2f#i>2I;j?7aC_G-Z`{1`J z*Npb69VXUnK4xZYG?y(W02D`kfkrqrdZUB9QXGeoX8lpGnNl+zQyeEgkHT5xbcHgj zDBhW42O9e24ob0ESENWsyq>gXAQ=zZn8_AX_Bc&LZp}sgVwEEmj_>Eb&O=NjF)GAY z`5X#jW0}knmiPGaB~zY$B8H<>VqWaK{HJ%!x8d8Uhu_P6UIb0(2_W!Jh}Uknw+Ghg zqc%HkQ5|l_HbqItD45P}OsB`cb}{7rNbx?t{wR_Ge%>jGGM)c!{rXztxpm``-?sOU za3?1^3|(X@y;`{1`0Oe^mrj4_VT&B%*xFs)u}{)h_;<3~S9&H_)qJW2R(b%@q_vcb zdzl*D?OFiK=D4d!Tx$N8z7w$TK*qeQiSn_S!_gq$$7>g&lYLS5hntFEO7y+^)gp3E zw4jIUW2e3hkvfy!bQKNC?1;ki>*MD?o4uOoe!_<{YegoHT!!_v&x1cW*nNJ(Su3S1 znASvWk2n-2QvDkX$drzOCYih03^A}EoC~a^jmpTGEddj!t(GtcM4*YnlCBam%$ z-uqtMhU)!rw_XO%;dpeD&7?TwrvJ-M7Jo=9?@^!EGiKqJyZmEltIy#t2gGSWn0?k! znNHob5Lx@%cXNd1IR&`RwnZ$2<*4G98P40s%)o)aaUOw2u2LzBP@APOkmvS zd8}E|rmV*}quDM{+5LsTDF*nG=Qxa)j;4)Fd`np%0rp%rJ1Jru@Y7e(4WBA;jSPyD z%{YtJF)(H{D-X}QLopz>#<*HdNx zyA1>eSu$a_!quzQ?iynLfyk!h#a6p3|A8lCTHMF#^I3D&urIiCgu-l+j4?UgetxGq z)({rWzH*yBe)kf68ZrLUdlr|4nYA6(DNQ1n*&r`BPfz)Wz|(vnV0J8(flI6I8-O4& zuDHP)9f>y}Ze5`+!*2Pi8Ad`<3Iq|=j4xElGtT&W(tj$^qo|W3Ke01Pt&O%ms70kq z@&{AVd`yzZVpNKd>F)BNYy=*HUa2SOsltl!sBm+^#sPD2#L9h@Oi6ZY z%~xSMt42zwB%K03P&k_Nbx5!{Lh)>(58$*$thAZ+aN_WLktJ!K%P|d`rm|<8eVb=N z4AA}dEp4_-6Ds3yAXb+@yn3dAoNn9{BqIpFjou?eQbMa1XXDH#qjv{qCxtAxD@IB) zmj|{P2_QmaR+=qhf5>ji3>r#D{N{O%cZ3@3=H7Vgi!qmr!&&GyET&!eep?LLxMI4# z)m`i!j^_y}0dan=fhpih;4!=6@88llZ9B?m;VnWtuTD#v}dnoS7tx<=? zQ>r_+mHTn}K9ohh^Q=F*bJ37Z=d5tL{Z(~F{%~)|Ik3FhpC!Lvt`$YjroWDz;$Rfm z;B*@6v^OUkNeq&_9v1pk+6FK{ul)|Ey1`r*^xU(0t4!TmtkdwJJNb%o0mpUsfrew> z&EROUfkA^s?{dUpsA+Jq*;@&(K7-g*jw^RCvG9|4(i#<3yfkAYvq6#nC+kmYkz|6j zNyB>L@q;8hAJ6cH7!(CKN0WZte%!LGUx)rmtiLzsHKEOJHbS2qBk?e_UuQDCwHVtf zDJ+{t88OHsXC3ynD_^U}%wg*{(QK0kM{Hk$yv9Zon;9o3$E7lEwyyE_mI+ImNl`z^ z(!DmJkJVOKvZ?GpaLbg@`t-ksaN3y#smUpP21$s+ZxpA;q3q%gtAu;e0frFdjOHBi@PE2 zX5w*ebYpjYK+1kk6d7_imFLny>>F#8x0;pNc_XR%3y-e*Th`WD-8yf$(x#&8gT;1# zn!**aY!R<3H|&|_KNi>`L!fmYA^e0q5Kwqsn*5a)-=JoRx|Alj!LKiGEpGMYInq2n zMIH#eImPadlkmhOHr@Y5!WA6Y+)WR@%Z{gJ4Muf+^nTc=xZ3nr(Je|&`B=OL@Yu9e z?=Pson<%9SFJKwv&M7WRDw6}~W%7KAJl?s6D0(B+EmqDBD2jN)702ewnL&`_AQ<=5 zqP{WQW(Uc!V}#x-u1r63%;y55u6BBT^#uPkEyM0KJC5)BYL=lwE;w>artuZZU>7Qj~IGVY!>PYA~O=L zd>HVe@i9cG00bPkCfupWVtrREzVygWPC6YK<5CunGd!(ZM05%geRyeCk{~Fwl2Z@QK zIGd48b<=8J6mF$xDNeEIvFy-kwz;!&4-dw$xTI@306v1-BL9MK-H_1y+Tzw1r&Ixp z(xs#uZ(*O4`i-}-(i%9cneENnny(<3NfBa;r?Ys%m)t@h}0 zzPrly-lrYEjJVs+!$bs9D!k=>B4em3cca95_+K+B`>e2V6nG_vV! z#WL>52>Hucqkyi1Yg*`*tPSmXgLy0xk9bYywh@iLEMXV48zthK^nCfK|Lf{TlG&h% z@#;WmEHZ^n+lTBXZZJ(4GwIAb>v%&CB>MTeUB}_#j|y145NqpX-H&{8TwgCjDQ~@p zbhC)OH8vgCv9%jlz=al~pvBQA@*EM175Cho{}!OjWBI5HP#Rdr-{bE2u3X2IuyywI z_pKM?jhqOp=9`(eTk*52{m^^AHFBbMfK6n|Wz$rTlz7_>6)WItBb)WMIR$LM7=lSM zb%+t6%^iP(j3d*E@-zF3Cbg)ZWZa6%e+bqR!-JIKv%a@^rmAXtS z^SMzCa3Nav6A-59$Dif|OeY*d_{y!pcf`|Hv4?4OmTqAVG$!2}DY2~f;_jd0nF$eJ zRXDY?r%yOP?SWEQU7KDzzJdb%$>9rkW)yj<AQ*e@$RPoZn0v*v z@qLU9>_ux4#!*B8V&b{F6lhhOfZUKQH*DpJ=PEvFJFtI{Jl)Jw(dQc5RR-up+XswR z$AaLhDy{~c5*M_n4Jy1Gwl;{`8GZCY5I=vtZu){op8L`nF%`|)RDh9GHjXVKk5kU} zOKzYEXE#z)=sGL_9O(7{zNNE^TGB$AmRw#1so6eTz%Rf#vfnPRZ2NHRHLS&2aKIf5 zI}zo?2kEdWIIYxpBUmAknR&=dUjax}wN3P|2bvro3Da%44BsQUt)-zT)34Kp()A2N z*uQ+WF$;Ah?t0~Ar&q6FBpb~O&SR}gGW2KVwF@&g%loEm{GHfn7-OE8HTU?lAGGI2 z`&Z-*OAwj_tBj$@HDD}W!Y?xy0F*-5bUq}x)f-dx($B!YGpl5lalBV*dm4$wQtdn=ujv)pQgDT4|409A+a)>x@iX5H`62Q%dowWW(+aN~RY z*~C);3KHZEbSa$Plcx;CSH&18ui}5q*n^POLBF{Wun8p$KRX_cTTyAvhy&~kGUv`@91rto_{$sI-LGAT^`nfC&5c_KXK?H=7rwfsTaY5CHAq)r>&8Yqu5I*k+KME|}Lhx7#c?j8omSebg zJxT(y{m{*4?+)EM<|Gg-V-3)&^&kS<=c;CW{mD9q!ZNe2A7!?Ivl)QcJ-9gY>nE(W zQ@qxRvk4h~53R)1g})kwa>uAGuTpn+3f~!+@~EPVBXMOcKOjfHT2}?!L_n`4%G&K# zOJxsOkJzZ$ixfzNn~B%OSyOx~f=ohyxbX!z_H^aXD-#79Ji420I^iY07y z{q2Fns#<|mMC}XDPa6A(_LZG)1^2Y4*w0rzr1|Qd+S*b9z_R9MXML-;`R}5s(szhg z2T9lSUK9gcEO86pbtg)~^hi!}A`A!hSJ1XCp4#Su9MbiaH;vU0&LqRI;pv=@ZrUiI z0)_ya*5o>JaqMW(rQfKpiRM@T3xDL#UQ0?d1lS_BHYS5_O|JtbYSU-!V)Odq`a^$E zUFhHdU|GWPFK;@{nCuuIM7y%(rj~={W&^9Kgotn3TM;^EKJ%C+2g~c!MmccWbq4rC zT@a#=BbueCm?qux^Vd_gjHqHP8?26@FsxPU>jiBqbEU^)w*zaNhY=LcY!#C8H>C!R zl=@OyJ`XoxDifJIs0_GLIG2MWO=wgnF>JpDeI7JzmVZUa>$RX`n#CEXYPsF4vUUbm z9U7+lkuOFN4InO=r#JFv%6^7nv*6MO zc6PC$a13iO>vhR4a)Ac@5Lr%?At&~7^edXX;8|^Yh$TUp1a=gOdW2{<`F%ytfE$Jg zm6R?>!H)5F54Z?aPv>)=S$iw64L&ezL^T`6A)E~o-%h60t}gUI=2MB{Go(R=2;CfY zCWArn^X7)t9Vp;OsUIO7@H8YGG0KD$&Tq<}Qr}G81fHL{Gnlkj2T75U?;kL9SP?MT zFwQfmgZ!Ew^{016C=%j$U z4vqVm_co3f0dWA6X2eIB0tC2|PC=*-Zo;b%GYi|i^B6i@RwBd7y!I|UwWWJ@@l+e$ zDg@Y|0B-ft>(tVm%7~m10dAF{M-#0?-NuKDS^qQA#fW{L-#x3Dz8AfNnKv|Tb@C5Q zYo`77M(fqNu<4Wm!`HKhuWX+Q8ldb{a_*k= za}`|xM)qy5vNPc;fDTFG1@p!V@+(WkQt>#4c`S_0`KnpR7=!rz6-dSp+*ZP25smMz zuxI|Xssop7L1rC)0dVTbF8M3r$=$;j{7ku&?gl&~uB!WRryqP~m8ke{{S2cPzdeSS zEQFr=JCyUZ=CI+NP!QjmG>Oqb}=0Tgpx`M$*8+S(|} z9s1rauKD$*URthDgF>Pb0N$&8b}uVLQ9}lf{R}}<_C!&me%C7$wdtfGowJcrl zcpu3e*JZ`p5tpyXqs8oRNd|r|-Hb5I=J~{EoPWgybs6Y&fEbyO*A%@)L0>FGxZway?c31s=L0;tn$*b&YZ48w zYx72#U1GS|*Gr!DhE4k|DjMZ^)6c^w>>D3W*3L2;WawdK7xB9tr#FbvuikMi2FeOwDphlCRiTrJ#pw!g-4Wj&2nB z_QQPwzD^3L{2rI<?2;-;85Rk1a7`e-8>~x(bikUBzp}=}2bl@0@dV8#SG%sYIWx zV!mghp9oB~MB&6D>^=<=%n<7Xe}~Pt*IVWofwp5woTG9y@bKh zj07GvS}B8tm$REn*UJB@!j6TJPu?A5;PJ$GnKn6If>~H#dN@59HS4g!q}_ipYx4P8 zU%=wRgKBXk>X>^)08ul;Fo)W`e?zrW7un63NJxcS7FUJ6njsC;12J8e(Irlwc>hWkOa z-i@+-P^)6BrAbcX0=>rt>`<1F5XrPOB@&m+Z~gAAdN)r5?}J9Yq|p!cf}cL&IqJJ} z{sfEq&7R1KH=Q#k#Ek5wLwV}~#m=b{?Xv79!xV+E!Ed}au-74>REQYs7ZZFJeU+J4 zBUjo}3VCp^M6R6&5mB3!=8t#z7gQmSt)vub>Nc@sYI2rD!-(`aH*SkL&-md!L1gf0 zl0I}`@k}8D7oth?_?3~V40aTIFt$>Jb)snjXUf8ZqZ%TvSknx1#^u@b1xaE%C~@+1 z-Nj}6OI>k3`H%f&OTQ;%A$pxlNC(7?e)Cp-gTU+H6?r7_$)YG%<|kWS^ieXt!SwIL z$x^|OX9Z|a%dbGMeo@|j-JEPn#6w~zsGpF+2Y}47A}G!(G8nQoT8KIB@jNu_W2vF! z85_%CfG;6qr_a+zgC(|suUbyj6~%tm;I-(t(q>@Dszfs27RUPMqrCZHh!>QUl=PI3 z=!H%OPOkXPSN`;7NmUjPoL;J5FC}t-m`@X|gr>!BnWCAPFMsiTx5)zSDA@YPWut~P zQNmWg`Mt;%xAVrQw`vXX6d9zpSrsLqyBVS@(`X>_#lm;!fcCA-N>X)4Kf1*0GJ{`Y zB7$G<5y*2ouW_>%Bq}hJvn6ivEH(ac>m=G{bTQy2>B2kidPibzi;+c<@%~(P_Zy`Z z<=cXpM>m6Hyq%G-WnO&h)O7ds9QczJ+$ZuVy>E~%+9?0pjFo#;BK{^WDslvN8DEHL z3|pRMA)=&_A0}?LT#%bE1n>(ivh0zLW+*V#3F>vw1KsNV^HKL6(Hc@%o%~B}z$z&A z!1-DdA2y#52K2E2cMS9lS~}9Bmd$VwvZI(b+=Ixy0{J7&}GDh zWhUIs2#;KVU~?1~8|L2=oB6MaP4fZ5duCr6MYEX^|K_H`o%#)C21$nk$;*OBGVaL3 z{az1?>E)7I<>?86#ZiH%?`C4_OQQs$9&1>`4R$CN;$Log8kK8{8uRuUm-taQ7s7&?%#Obh2|bbz1ymrq_QGL{O)1lLP@13kp3n1`O>x$Kf37yypmyBP z6!c+CG(}Flt9!0)e%h^%T;!LzADP?Nx1`6l7XDJ=S+@L?lUw~*AK@eWv~`*0Pjh$d zbz0kWJ=y3lG!hm^Gv&b=K6QGI;g3n)a=$4cbu0gKbvQ)o@bw6 z&*J6BJ<@g_JcU?-C&Zw_4+C@B7*_ozo= zwUz!)OiVijPn^WVCBkl1{BlZzvG z&=0-))v~T=>%rB!_C99{+tJ~#>I4L9Vx%$u1Z4cdht}n)I72~7TbU>=`QKWJ`%i1a zqWl|T{0Hqve@dEHU?UXgc&5$<)L4q7zD7z-E#ID5Tg*2^um?b+EgQ6jK*bO&Pm3C8 zH;wB?Gb_aiBKdT;|C#)8?>Q3Kh}xE%ofUa%zuu!Z`+lA@BLlccC< zRFU&SBDO@BVo8|pskeOC#&y*f1xR0^^ zn|{eeKxYvNu@@ceGQYi5@DRLFJ1Ox-yBxM)!K&-_2`N%9OJ-kv-N5TxY3+-ehR!PJ z>z_h4K%b-nfUTbf2_NV|cd;t<&+*F9ZA6K7w0(LE?nu`eSATm^FzEMg5Z?wvqLSV6 zYg<3xdX{ms4@Dz_f&q_NMm|4C&fPu8?@c$u{@OXw1`o|bU=+hmZ@!j(<~h2YFFAp2 zyvSxd!b-@?8%T*^5zWo2e7e?y(}V<{h+uEO){XBqv}`{s&=IiSG$c(1Q{B@YyIE+)D^U%%K z*0$ZjhlQ%ZCAr8^Hg#pwLHf|R=gc>nIgpoQ=*t5G*V_mKHB1aAr?3PE z5iZPhg_cTp#EXQ2ti+L(6t=9%?U*6vcqVW@t{5Qf_}DGx;{gmrx&<5-ZRQQ~ulYk= zdhV_^%WU1h1-nlU4w+^B?5Z)(Rtb?@SXpyE^qgb^hvBBwNX7cBl%1O)Zq*FbJqYAR zO)i_iHr}Sw^=?TVohl#{P~afOLfsyk!`=~32%(a>k!h)~T>Da}wNw|DJCV0mf2g2% zSy4x#glVyLqWq-gR!FN6n?BaKQFdlJ%#xp{ge-f@oB!8tp*;J^B;)+>{_;}^8qgM` zG{?YWh9$r^5mV&(BtMCm4299c?20Ym_+ZRThL|k%xHe+3o?99WWq9GDV?Zoyyv0+DI=xTmFAAg2xv zQpcS(qxYQWlDH3+w9PwGlds&QCbi!5DBU1CcJ(VgO=|O)&uVKLA1xI{;iP z#4BJ9AOjICVM^ov0;>UNPqLGHpqbZ>7AKMv56lkhZ&Vh5Whwk9iSyeQa1U+a1Au%I zd?Z3V*&vBjDC_^U6ESoT3>(THX_aMx*xo*y`|W+(s|r4L0?lPD&KWvKUa694%Z z;}uXE@yS)GAP1(Q0e~9(2by-@NnHRUMy`uaN=WJi(tP;ex~1-iD7AMOC)8r0{NJ(x zApHnnsaQkeG9)9Rw#om%>3)Ae)VoJtvq1XP1h4-GTuFs73`{S^Fan$Ds}b#gb?QeS zDFlIGQ3Qi3){4T7&N6FShd2L`grB2vkR}5xfVI;a}*-UxlUIU|8EAql&L{y|k;#&ZoU9i!LQN^wN3AphC>F{A55Qt-%;;f+0#@ z{)0CKCZbsg#;CF&idRWNo3mqAe%vlRY5y*i(D%AFyaZ#Qkd^9Msp8Z$Iz8;)(s?hS zC>RkDPY!CO-r%jvFWC5=_dnbO%U_KutUS_wlw z+48^ZLzei^@Z5kW&{u|(Iia?)w|n}^uKd+QZ?7zRY^*R(SeSH8zXkDn51&UY!7h)+ za1-^4z8{)IiYFTX26z6F!$1e+0Aj?apo7Ct*)pWQ%)u5V=r+QadWvSer)r>pr0SI_{Goox<4`{Hc8X|U787IR8kS&_v^U6=lgXp?fH_&xGX4$ zlRbuT-j5{2JI*oCvX&4Y85BZQ+58s#AKNlw6*TFKbwKMdx{-_jupmB?#P7OY-`CJn zYa!%x7kR_9o{cQ;HP+B`MB}@z!~{+KIJ0l9{j^r$82(nuQW^}l2~FhVN8mpo8~ zCIJ0Hav@|M>2`d44anpfy_u)sq3Ijo2UG$>z=&3YVKceh;r}QaZe!btn5z7oGNbu* z1rcA3(plD;ERL{P@4sQ}2MS1)4p@lxpvq||J8*0NdjtJ8PYG>s3G=9+@GkO+miMW+ z2Rlig{n9`Hke#=3IBzMy_j(%1L)fhnz9SxE1P^Oj){`oHxbzEjxN2ZIs}|l07C6A^ zKBnWJ*ac0Fc6Yf}Nf`nB020qn8;L2L*w`Cswnr0KK79*MyRp5PPP7W@PIWH2f{``g zK~SNtJz+jJy?h|JnG|IMl}gsaZ2T7}e0m6ve4NM-{sfKOH<0iIe$rq-$c(I>14ezn%JzF4aR?(!?VXkU z#YGA7utQfNG<0^Ks;>lGRkQkhtDEjw>5Y~hg0mp(K;FMsWt6JD5jgHC5TG|oTD`U% z*}ZlmiMVM_;vFsw>bc1RT!*(-1`VeWZZzS8&{f-&{~ggoE*9|o{`3=z2jvw>7@R8B z4{cPL-QFziVV9#K=Edxy)M7%DzG+`1%lZxP{kh{{XxlaKrPbpjC@)t+S(xH3_kNDV zfCru61RIs|M z&De+*S0KO})GzkRrOEx;oy8)hUGL5cLKAiZddE752wtsO{p`!%K-;vB5P<`td`^2NbdJ55y-5 zE2;c=sHH^TXrJ|A2C5UK-@#qrzD7li6%HkbM(Y9bG0_GFs|G50B|zi<+5iE}u&_Nl zSTj>bLTx^mWKTH?e{78 zeNm9r->Ww)>~CrRXdS*VHb3aIeu!Yiy?*m4p+@3gr{K@y7}5W}tiMSQ&M(A0pf?!a z{<%F8;sNa@!C`dD>_^$aKBa)$Kdq1jF5cICF_a$st1A*|P}YC_ta2Y9K__OB^0U5# zOepUkffGOfyL2D6w&eJA@`u=v1^(J_zb+m!H6W^lTn$Z7@X7rRpO67{{a3&N)q&8x z_CsPd^ur@Vh2aI#{23YXYeT%dk7#*TS)=wtB%48#&VLO)@IW3=e-8NVY`E_)_7EKC z?}~q^5g79KTA4gT&fI-0UnywP{;#sY&J?-Vc}aQK?&CTFN6Fi@|D_d@CEx433X58R zmMXSN)vo_*Vt;kagBq9<4`Tk>OyFYYLaYR0`hKt4{TBQH3@hS;4e$iK=$ zAqlYp3Smwf(J`>sLxiBe{|x8&qo83}-7hBhNohKWf)x5J{kMSpRCvj5?q~ESmRLs# z1_;tP{f~+wWqhyaE_YZ=P!d9)rTx2X04OIHSgnzfSbljjQ>03sfAt#ZIHL5uo=f4Q zPTYW4S$Ds{81P6ZT(&`+v~iFWUivh=-e30n`^AjDmnxg~skwEU_UIoEt0JAS5#R4$ zkGZ=VR0K8>Cg;pZ%=@2y{9&d7B{1DyOjQMdJfweoxfm2+%M}0|Bohv?^2DeJ2b9D7 zsGS>s^cD)fd*%58N;ZA3it=YI6_H}B;Xy`H0nUh6rF=kYV?_7IaX-0L?iV3YP67}M zIgZiSb5Fki6jTgzB#=9|xWjn(RBVmWpYC)}0*iRyA^rX94(|`uQ0x=U;^uo8<#A~n z?VaDpqLkYGj7A1p(Ebt-G8LGda)tvgy}T6m#NQK4=5xO+LeYdXzgM#&M@*#zIVjTq zl-U&ZPn)k5fX-pGk1+h+AVe65R<7bIhf8@;HWa_L+aCd&AV_8<{A1MyegAvEa=(HQ0;`rx5>2`fr z!(4;s!c2dJj%)`5v6>nPq^z_^Pj+_@j>)x4kN@Y;WyBxFMLgD_?`ep%zypWOPjmE)f3uk6Lujp^{&F&F4f(_(z=)N`cOmYHb7pD@74jf-;6uX=OxXR2Hxz z+2BE(Y?_V%De&N2p+ZUOw35@o7`)ihvTH*ypgx?F14S zGQL-SwD_qSUDIx{=x^of$GP(3W&>{Nyb*~CX%WTdwgiVGgL-zd9ef&LQqlTb{(2am zVetllBP=T4Ifdj9;50+S2(=DDK;*(z&Jj9f9no#PaXUfeTbIj&W)D;YB(`t<-GIdw+dYj5`09aXkLHa7P=+qFY49x_4m z)N>B(63Pm0PmRcD_SBnE-%~e_eFr@b7GL}8ue!U;7e`HZ*9+w~IcPC(ZP68Ox6+sJ z*f?jPOmK(t_)P1#fkZ39z(l4Db+pe4N2U#QHVZFSbz;Yyc3SGEy4`zVPC6a2|wf2#0io(d%q};YEv3a|1D#HL&)HTM@c)ugR=l#hC;m=#@2@{0h58TT#zaBAm zpKb2m9mYj@I7JW)) zD`RHuzW7%~M_#*yluUP+@-+{9xP1X18MNF+DsBaEvO7U)DixCP;MQ)qfy@_@6zK2g zq#_*Er|tPCkN?C_0w9+}C_!m}m>@v;jl^r>X8YVmfq}i;$fL!hP;}}7+@f?n@w^#l7;_Gsl4Po=Do1+m5d{O@z zAUTWNTb4N%U?P%&%~So>6c)Z=&Y)$DJ%^R!b=k7w_wAT?R}H!F#OH8L8PaOe@#EyY zrnA0lY_ZF%beb`e$LN(kGF6~=H0h4N0v{^`mKKAT@_R>Ck>2kpPT2L2T9$w<#@#+x zPgn2WfwNj%^zJfR#8*bkr3ufhFN2Fp$Bu(euT8Hz0vNS1FrQ_^(v-y)W}+buAAjFd z)qjX%qwe>j_W<7-ycHuzr!Xh+6g{2H?(5E})8&;dYMbwQD}a(#^&@W4^dqg4y6mVA zlykS0on4N(F5C|>n`VuM#rx~sk<1Cx&>q;_`q2Tx@^+rmF;~XaEmyNf^MLfS-Fb`q zUBVV~>lT1>A3Z*9Y_KqVm1UkoEn$?5fe&2M(=HfR78Z zD!BQz_NAels93>x_YfST>DZ98((BeVVEHEOuP>Z8&U{4|V1$J9>s_SLh(yYZS9&}L zG+=}HN~ip!VzXPATnE7k)o`4HFP$|^KgUw4F}Vx8Bw>!EQo_)Aur8NBc)MOcMM;m< zoic6aY(4uT|3(6-)pfmmU$MHcr)DTBt4T%uDDutA(sbPS)o0s;Y-(co01jzrma8;)M+ zSa_32>#Q(Uzh0l!>hWZpLVOsdpV~Jv-Db9S*0sGL{H;*Ni`A32hXH2E5Dq|{2z7bXnbeD} zz`xb@dX_2en`jAr!(*KHfmUL2Ss%Ncc`2mt-ed7vYxP64k(ps5eb-JbzsA8vKS;7* z#+g|nSgIBD-3yvrkbI|BUjM;Hj2@<7uLb9SqEcD&HpH zdf-E5_9`49{i9!=-pqyBHtoKHTDMy`PZMYZ~=vS$o>pQIJe#nZ~?zT;nepOY$x=YML` zZ;3Ef&V8HEK$jfdjRBs;a|oBC>>LGhRP!tc4>r7KszeZug>fHChu^$Ln#*Mm!{ z@P3_-9%y<-mA8xZb1+rceOgt->OzKv9mU8KRBl5XiR{lQru-H^u+x;LAfS`>W156i zj^}U=e4KOAFxwaQ8m1eKD+nycg^viD8k0M7kp~B`Q;?CWtIal?i#0bhd6vnkTbiOE zIRcEgZ@<38WG+tp1VE@eil1`6k_=0J34~_g^RP8y_vL6aWC&Cj$WuIzv`CwA-yghW z8A_Z$3#-F(6c z{lk7j^X36U*bwvOz~46$83?J80eC?{l*yCT+*9^oBcl_`U7u}*x{y9vPfQFw?iVa0 z>`!j>$K}8dp$hFN^KaK zu7=4oM~oZ8RT_y!3c3&(8*hGQ;=V2o#+PH zlec?76%y=Lj+f8-NsTWrzTZKfu%l2IqfyMWAX3>|p;i7`4{b&aHrDB3&Bq;V%76pm3=1dq+Y5`C?m4 z$I%&2n$g)p4S_bE?R(`n>kF-HzMt$l+-*ehFC=-34O)Wu>SV{P@OHBmhjo#*sUa3(7fF}D31Q^ zla$$&W;Y3@*Y>I(<9$XyWIdx#)w7#xrL9rlt0g#!#PscQ$*xluP1iaxI0+I*3(jQ~8h*#m`3Gqrwp%a&kHV7Oxc0FuRtD zd+P|u`8Q6qB36BkU_@-kdZm~_h43rtLaCgSMqyZ;2l2yS;EI$4bE#9ksvg0Tc$-1p zOwt58H)11EE0y9kw!ls}XqG|cil3tW6bix=MQYF6y_Pm4XBMEarY^>BHZjq94*}l}S=mmGOi_EMbFMAYHTx ztV5%Q^?rjada3B4kc}&RC#Pc@S52dUGZa!$Jpdj0pzr~5^4(2Jreb(G5NHY6X3h4# zi={eJyu0CY6#72ra=y*<@wx13!LjeEcDih_--&QIjiXL?D{t>Z)p`#^-J;rc+U zqZZbu0;+_6j8^%buU;!SelZhSE*PDmeA0_iKPypDV>)g#XAA$B>9&D@r(rv}=HHW>!lil#0nm>wg&=js_U?mBaP02G zr*8nDKO}_2BgiV#1T)o)U%&B0koM4Nd7a}aM&P3%8y@U^@g$8c1K4`s^FW9!{Oi0N zcFwDnHQPi5>1w3ZWs?R7=}OwbnlEjE-0+91GEh^biADJa58}lp<7veJXjOw|1s3m#pikpcQl(sIN2wmW*| z78EJXjqeVY-X1Bh8w!2ONZk|phdr1!3GM1oyBV{g8Tvw&&cwsPk7+=;f+8XisYm7DnIJTd8rw>tK^NO!#9~d}Z~LEWd35 zV}G;laWjNpmZRjx{=8I+;sz)Rnu(#eHnm zP6=*%L;bGywb}=K+Zu>5Y^MUwaEYs(4i{P)jFQ}Vo{V&h{^f!J0Yp3Fjn^#&!cGyS zUK-9)w>Y5rd0mLlWNDTmHPnDl*$W;|;N-GYYsk9`4dMA1x#*QxT^$j-DX&%+#c?N5}owPX-ivuL`NOpV>rShEJAVy)njcwajgvAQMvFox7tm%J==Ej6oiGKnKE%-%qmVQUbd>Rvb17V4=Jg;qsFEP^iU9qZS9>gAQ5MJ%!t?jsL6gdmnv$D#YT?Msi z8flFXC{=zVk5zVAx3NkOgoe4yqJ*^;yi*9XEpQm_wf0)!2Tl4DB{HB=H)vxarH4yqT-(o6cguZWs}?oYvSs~Hoya7x42yo-OEvF} z<2WodXoDb*LIo!K@AQWXuPzt{-6j@C0UAynyr2FB3>>7&C%;Ua7}fV>wB}2H+!sG9 z`op3m2D5Z;$6m5&pVa+hg!+d?i7c>B2C&%n@hty^d-=nn1SkecK;aHh&*~pD>fbC% zR2XCTO z@#~bueUXS+(4_Z2WdR~XG7Ny$&)eb=jsV9`QzoVs{3P!H?=QRSy;p5X7CuK(8NEh} zA`2S2pr*}FYoqTckzj}_{vzkAHq}24LV#Nk3qQb*AVD_-Ftisy8ynm&(QodrU+WH7 zb!hKB0qgZO6TpZQ(xANm>M-!l$UPC)dS}n$`}x4o25J5cQv&7#4Ja)UE@%yKW&N)+ z2(jpXK8j#_&4Kv<9{sNHmm>*iE#-bbTvw;;u_CfSlRw4!kI^)6ok|SI2^VAnxKjd0 z$=U_}w$1_FmxuyNzn0E9P5l3yK_NQ#E|;0tCSCGmt4NhHe@*ngGo$aG0<2H)Y{&DL zAN<}I_~)tsyzKEkbhp?>L^-KVx@r_(KDS1c1~H`Zrz zpkbfmHD=F?10&F$(Ltjd5Ng;GZokk3pvRqk{L{ z1In&+gziPN?v;k^t*Il5XuXtmOYa2Swvn8IHrMUiEJ6Q{q=E{e&zqbU23k4?*Ff%W zqXw7D>fT#1lIXHt{7x;2%7D_48uOY4DvYiBi9RJagze-E3{^~3T(=X3+3g(owr#q9 z7fccO){DRsNb6PnC9m-2=voY)?Zp1F1rlGB@Om|q+oJW^d;R^CZ?U*7R;~o{^PPoe z-(I;UR=mg%+h%z7Ca6@m*c}-5Wq^n?{e`Xy$70dYz)i@ zKKBb=?d*=KC@8xgx=k-JM0%OYRkkzdE@n)vK;3eU;!)mMZqP=qJ`^$ruBpgjgbemQ z``|(i@rFO&g81F-W*8gsiHVBdtR7n|>vI_nH-`=wUJdb&V27q14{&b}e%zkOeJKfr z&3n+r2{qB-xUqdKl18TUWtj%~L!(~P`GYM)E>$+lbyi&pAc@KY^YY4EpQ}>-u>!Lu z3j%-wh=$+o<`d_VINW0EP`t3eNX@!0hjpQ8>=8H^y^VN;2hi0RrORP0=sesin z=ziB2r%&~#i7$E%rVINg*i6-St?Xph*`+1ZCl2x&_-{Gp9ON9_ghp}LcNkB7^o&{k zoUPb%Bh1yuvIJ)8lX|?UY&dk`Z#L`?QK&7=xb5nEV^Q8yO@3Xjhje;v8zjb`Eoa^r`kb5WYnYd!RmE0PU?CdapHyrP z>bO=QeeLdJT;`;=WrFWBn+#w2H(a9=CAy35(R|zDPKzaV`Su*aIa;mRa~lm`*e|hE zyGh(`zLDDC>_*&*$A+GHriC%$oVDFUDKjls+&?yX++tW^A<+L-d=Vs-$$%PFr@7ek z8ij~vqFgTx%s3%8!Rl@KW>?VNIx{bt63Uj(XEi-Kn@{?Vh#g*>gr82Gdf@q&sVN%f z8IhVSEL-cZ%^!!~8>X6!%-?FWqz2dRj>>njQ(o;SL%42==6zSkS-lVkK0SUMXYc>j zp|^{0S76I5AEyj_kdS|LZ0~zIt*1v|iKeOjGQ)MK_1QaZ+JM*aoC8xsOo~$zRs#ja z<4>$6Am+n4w8NaTZVE#z`K$|f2T^%h-sgLk25f>MQ|~P~%`L}5qtZ4}9(ov=Bl_6Q zHI_N{MY*+4`L_9hBZu+auivK~Sy`EPz@ywfRberI#bZDB){~vl%o%p3#N17Wk_fHu zdM>VQYv&R63k}b9#G}h4+PUP45-z&2r0P+(51YpoZC`4G(ald17P5D4gsq~7J&dwi zA;oouv+?sOT~W4URPt2*^Fm`ERDCt%b;FE`ZHpg#Tit0mW?9)48``?P7$o)L_uhkP zaM`S8(yo5opv!rVeHEM`JY z(W$WJwJEo_h6=R_ReiDZh>noQmYuW5_Ee;Ii}VxuZvWGw=P!eZ@=@_Dm?Y7oQI8s& zeG$+SQa=+|b6w5mSu*Z&<{+hmyVc9iMQ)@}N%nFcy(8qTH-vl_Bt_h-$nTA{&N%DH zemrx?Cn?fFy7D0OfssRbj^2xHi?t~Mi#v(SrL=|a=LDdZ4P@iCcoh!0zgx$1p|28}GS*N9NC6H%Y23{za%$|*HER9(;jF}^hX^^EUms{og2dgXNGZ2?guN7xkk^PKvC&1i?OMK< z$OOGXr`jfTv93!gTFd`ruC;bV^J9=hAH^PTAM&y;`{mAH&U>2)+gUY-HuAYeBY#GO zXx=Tc^!ynkKTe~`NPvL6-SzgucXQ^t7iz}V&`^qCraP8c5$jW21&eyACKj_c*C}AJ z={?GqjblMTBOZH%2UulQ3g_A9&t^FH^hA7=-$=cr5D{Sqo0y1QY(CQMf71o=!;Ec7ozVcPbBjn0l?-n@z?K z81J}7_X8{l{8-yjoHqC_mm&po6yrHfNlnd<4t0Y_M=<-A3J23p>+U~2$}u|Ihcm^n zRKXnxD6M@lr@#Nv>xn+^K5q&5bKL@NACw=X-rUo7gL5q}*O77x-@%?^L+FgFO76ThX4D{Pde+Z&SY(t8GoSRgwuAoLs1HeJ>}MJ+pgsQ;$X#zqN5(@n#pGbGk7EO?Bl7xsVA(|BKfii?aT+pZYmm+DSQ#C!zEd;&2Un zC9ZGJYV_{}%|p)2W@59wcFS1+@=H2lSa=9^SNKWNNTUzkS>osHrn5fUxz^NgitGIW zqJ9Z+O@8F*V0NvXOZUZ5Bk?N>ipTBwy>TGc=7|0C8D$RFD<<)5Wb-H7oKp6Rw>0+e zmOXEKTrVwlK%V*1PqkW1a~4Cr568gOZnrWWii10r56w1jGLF340vl~kI)cN4%!U?g zP3K**6Nvb|vY4`jpA4kXQ{P4h_B~^jhFz@xZ42yL{&xQb39K(Q$J z&ffkdDQnK4P_EBOtLad@e%J{_Z>;di3J2=eaRJi@;@e09?)NtOP0#Lg>2!s~Nx3;v z3(_j(B%+I6&PJ>HuUz#XB$$9o$hm@}U`_3|KKYxD)ng`dE3R3n4?t| zvo(>T{jKEvg5#MovWs)QKO`_UDs4AdW?rrEG5PMoZJ$FrRko;)Mjl>Dg_@3`%T?J3 z)Lhtj@9g^KXXiYVqByt#4kA-bj$kVwt(&oRwzH5$UWE9RSCOwEy=rqrdjQ90JQ^62{6~#^Vhv$g!PIY5)mxhz zOd+wTL60zDqradWh4_)r&(#>$vx@;axO7rkvkH#{r-ZKU7FvUu?uL`R;Rd7veLPLS ze?__~{lMAe<+ju?bsJT#_JvN5wDr8UULkm$Fd8Plsmr@ou0;UIw5Z%VYOE*RWHs@Q z;af>FO|IKz`|LV3ceXFyr?KHA_U*7&UuU(?(=TiRkhDHQ- zzb6YXXocIhXc%!mr@}?0BIZc4DG>?yd@3L48%LT`3tIpFsein{vCE6YZB@mWXMrPY zf+d#~mqiz=51OOF7+EV+d@(6b04Zl1@J)%UrS0{^5UyznX*|H`W3+eQVv1>yQg1xAGB^SX>-7&qPnAM? zXoXa7vGzHg%D&c9uXXFRC5)fMkgeUBDwCb7tNmyJ$oiu`qy9c`X0tKff@KvRt&QHT zo_1XV?nwUC*Wv=XVuhn#R?&?C35`p`zV`GkkAQ3JNN)92aDU3vgRGCorECx3<9SQz z#F}D>glWE&w3^Fq@1ymgzz--lx=wt<1dM18j=mOa&505Vv8fq$f6DSicG%oMxaKiD zFt{ppyLsYD8wDiuN_d&{*2)iW>xbcA8QZAM&o?&%_UZ(d4}=1B&Num82G_D35UkJRsF&m}dCPYC&* z`8RzznB{kFJ}Fb6ON#{o#&qD_`e-ApuvI#~wsfIh`aaRmsg(vU9{hWavWC=EE7RPc4SNptDIdR3cX~4GwW! z$N}7oTyYn-b$;VG$7;!9Tj{u?Ks^e*Ze^zPL4`qD2~;m1BB_(wa6OPdqzH|EQ~hly z$TmfyCn7NIq`ELt5gZA=QFw3@g3B{x+&Xc{{M10=a<^z7>}oPv>AXY@GhhIy>ZfoR ztg^J9qp8p8-R>6kJok5b&2}#2-&4IQ^*;9zojg;Y;iZaWFJCl6VD+RWLSdyNRG=JA zhwH>=scghR4^KiMP!s^K8mbk{UB z&wd?YGu1eUXL^Qovv(#4Iow81WVN?5fqF%uvQ>%w=mAPHiuc>fXD^M_n#{TTvl+vm zy;m8=OH+D<3l4f;%B0^YaksI4EfIom-bT}hute!iK?SUH29L3QNOg3T`Kqc zdQr(bhNdz64XOzjb3ALs2kTpb>LwS>*tm7v9=qf3wpZCfZe=Yh)sMV^v5 zmvgw^EeiXdx$@K;$3GE5)?+_puQ}@@L3ub`y0=BIKm|H~*8UkK=N|8RGut#(1gSyr zp-&fH*GEf8PCH&6=31p~FUxULlhoW8xa5TQ^C9m;r-+jay^x`HGXI&!Ug@QYbO{xz zaDFpJ*FCUB7Suf0JC26j+ z$*fZ!!!#&c(J&tnJ=klKh2`{o%H}UptXmu;5=G{YC)+2yC;K&YA>NP$lzgu5ZO%!()6K2e^{Eda-FUGT731=U0p+{TbyGO&zoJY;^k{z61?jVK%m8@9?2TIuj`K&^VC~2REdL$d_#szbXz!pHGMo@Ek<8PGb3*(1%bS=|4!z)Uv{>&B=cOVS0IE z&SnJl0rs$px$k^W*c;4^j_Ty5e~m&7I>%l`G%^hT2Bql=V*a8gaS`Zt*-bv3#9fZ!4Y5h0m>`)~v~2-s*Q{N~)6#%1j8S4_=@T!`@8D_ z^d3?jP>}I*I%yBAE?`hv!fsz`?Rrm$_c`yxICQF5vI$NHEe~quYE_MkFpz&s}#Gc%mwY(8R6^JvYI?TX*%!~vgoOZg?vXN@)P zP9lMpqcz^nc%`fjR&>i&ppfD(0@5!I3BRtdRF`)pnZ?t z{hWrQ(;>9ROv}pQ@a(!pB7OHj#Dp+!hb;#%ZNtzW>^$W@IEhBh1x;?HN#*HoVhPN6 zom?Pgm+9E}(W!Ek-pIz1qyZ_U(%zzOzal+TL{ztUNqxRAssh>iNI3LZk=XLs>#64n z-wSGO&J4*6wQg{GN0+(GMzo>U!B7Ko!3TDtbUOOy&F_YE>ZPjj6@|^YNJYI$XYdqo zn4@2aH=z*;yt&Im^H>_N8psf_E@x=Jz{_0036qkk>U?bDbXR<0S?2y zTqj6}uT+BzG7fXfH23>W6a{Eb#qczRG$cKc`g;{6Nk_jTNx?D%#YFDW&(M~bp*2lH z5y%WN0B_7(N~p(7LsB;Usnzy{p-i)FNouwE9K55hlI^XkYB7B{m+NxdCT=I? z>gI-oIvrpj_Pz)Y&h{mcdOr%*`K_2|Z`zJkml2oHR%g|nxXmCbzY+Q-pKk9^cszd8_nt{g2fwyzaeJmFra| zm#Jo(??E>BK|*9`r{cAS@7hnJ_~;;X+XLUM8(pEp5R(yLp@(D9N=6Q54l3~dLxPd- zUSX)8FHNTeW0t7|-Z+5=xHY~&cQ(E{PlOHzkV#Y#0pt&QSnzNh_9=(l_WYj0uEeIe z*P}CNFjcHX+AAAISA}-4OkDbO-I=TV-&Xz1Tpk3zA;NIO?G;{g2v!k_hb6A>6 z_1Zd`Lu%3EQ3sK((XXD0N`=II_nkOa1d@fe(^mxlwgoi$Om5n4j_oq`Kd%glCb_O) z*aeJas>j-?g%(5y#$YqgvkgZL+<_{lOU|ooJk_}FeE?2cg7c)Do`kl3`*mp3WJH_%reu` zra?htGY>!#*$c%jD`#nC+fB#4Y-wo$ah6T;UTepAy)EB4>KgIi;86J_C4~8zv((v8YOTi7o;=sb=lK-301vGh5{gbvvZa>tjE$D^ zt(*Sf5o!(70%vCx5drJ2N^_mv`E3b<$`11&Z;aVx4!~xD5NdZ-P=zI^Zqt!y>Erx5 zLv@Tje`Yr+4g?=*Y^MKI5;`?cGL}s-64%dX(dJ8{nSkW3Sv1W%8#G#tuTW_=_=*GMxx8;3%&(kMBs~vMGD^ux87iPKxdf=Gk5u^pFnOcru+}$ z7yV>1R* z#+y3A!4yLG_PNY3^qCWGIyh{KYxE=6WGGL$Ca&~_HVGARyKN5llZ7t{6!1SNH|@(Y zNPPR5LKDZ1@r~cqiK$lW5MbV9jfpmTI3mdwb_nI9GmDpG@+pGlFsUdF@C0tw(tmz| zR*Qna&j|r2fZT1!OD2QN7ADU3=KN{ehy!-9`^U`3FhSTh`)0>nMCq!HNJ^I@7?!8 z^k|T~d#qyE-IW*V{h&>8Lok}B#s5m_Rr9=hZeVOEcpyWsav4g8?w~{Nra|xK6umFe zL}2j2B{{LB?wM_&;o;F$g3`^U8lL|FalU4i4~%K?)*3H2+urkA zb%JF2=Mr`JW`I2mupN@*>0WnM`&=(7saNSCUmils3dZ@mxhpp``6rB2`)nTzlJ_lW zrP>okTVEcnouL`kltL;vTxS;&dxnE;w7Dz;6gMxs zX^+kqMSwK7I%((N>zv%K?S!tu*=$A&kiVDJ=jzKVW@tM0sI`mX$t5_C@mJKA$bB{3zDY3jN`ILeVHd_E#8Ad;`qP(cvpEiacc0uzMjSTAh!iwA*z@(g)*!dV z?t`yOdO@NMA3vWUDNu^Mib!Kmj{4w$*em;CG5k%0@7IGBI6_j3u4Ca#=h@T?o&bGCW9iIg2N@j#TFnJWF>M=LOeps+_8XbT`Qt%|Xw z5#AvGWvQ1gZWw-jR&PR(o})p|KTl};c6og~B?u#>Tf;FX6@^Ww9FYc&pN zr`>FH3;kn1x6&SFgZglfBx4UO77r{oD_ovb&d|*Rgd3UDvysG z+I(4wbc#9L&Hj6<1WuJSJsS^T>w25D_0>z=viBmHVkr*8sfFgz?y9!qrgM08R4d-qg2NJ;nyFvv+`j1SauPFgduY zy*^ktnO#bv{GfEI(XJstE9_+RlX;j2U#~ZYO!L!lcS0q~Q@`L@F_}Xl5pSE^MgB_b zV<6WqpJ-h>EIes4|6-iM>Tx+~SIOX#o^UvuJ}vt07`o_kL)f)ACm+#)1?)tHZo$aV zOq%j``pg#<{i;ONFn++QVlk2`ZARoPGt&X@E8fgVgGhRNj?r-&!u)W9HaZPaR^$ZL zPc}_Z1MRjHFu0x!bvvMtI81Ml7Jycswm%TwnMN|J= ziG}IhQK-}5SMiP-gvdr`?dX>!3(pFidxhn6;~rUxDj(7jY z_hLwWiJGYr=0I(2*6rk>0{)S73;}n$J2b{w^YC*$H?76UNAN1uJ8aj{agWwE*m}#bwz{=lySO{W-6=(b6{i$;hvE%d+}$05yO&~Z zaVhRDL5dW2EADRJ>{{=$*1Pw&e~}y{Gh>c9^d8rJopEDp$lI{n9?4yF=X7z5Rtl=0 zAt*{u$EHZ^8?o$TX>3Nn>95Buc8f|aC3GBhtb)B)4ZN#9EH#NXwtL@Hw7hL*T+nqh zse*OhbpYt?AcktOon-<`ns_X^j!q*d_tCGJ3oXcinWYR@%(tla=+TOr%`*%d*LuV#y_aJ3`g@m}&F4;#@g zaG51u;&NUzDQ{r877dIUZhAcVMTF?am`FlEI_;f$|0|;_X|jGq$hx}=a!+j^C%T`I zH3vm5mjp3K6;o3vShlC7rthem<5M8S( zM}^&}NHg90BkTc}+*|6T*l`BsbB|2Ko3D&MDimAD=di?mS0irmn`HB~awJ3HZljunJz1>(BYRs%5xeU@1Y5e`k8D(v|B>A9Pjs_Mdt+GV5YNJ4SV*yh zK@5BAVW|RIr=G(7jhlhe*%+-!laK)e7!iz1tj&;O=ay*xkN?0w`pSQ2XmbO z-;D&^PWJ$R_5b)x{O7JrQeG_rYNC+8Y9iAw^NRmhUCqK10cyQ9V&L0Co4p$U`P#o< z{3V|PP6OEQ;b}r))(2xiT%TSPhygZARv0SzKmH&8FpgCayQ9#n$dmg{S>Yx?gt_x? zN5*YXsd+>$e?(3o!V_U0c0uERgqU(wm?U6@#G(WK9x(q&4*#oqWS2#}^;3MJNgR1~ z3AIjbcq`Z@yOT3dBRvZ5_UHR;W&IAawOGKJ@#Qn|XKtpHwtTwCB)R3%zM;wgLo-O_ zgJW#{uQb3{E~8}3(ch`}inr-sh=ea4;CX01$iM~J3Nam6@OyAN8r%IJ)qz|KKgR*I z9*z&M!r=epgCRK7G!-8rfK~{{H!e4((bu>^g}y9fgZjTjgaItD=)ie8;2Tp#10xmE zhjW^R3_Q{*DEegc6e)aaUIatL;lEYk|89R`M6{hde_M0_8Y~Bm5UzZcz>1_xBSw@1 zVo11oeAPYGQYvPb#$>w0^B+^ z|Jy|4KZI;51w0S3Vm-pgeo20~Zf3Apr&GdOfidR)y6yjdb^|xS5cP?h=8B`67@KBt zt&07>%>la@rONT&Zxf>awx!RF^Sb}Niicc)$p5@U{;L%Yl8CnY??VBs420cr0DUYGefBfylbf`Ps{$~UbqXSGQf}&wM{BD{R z_=QvjY!B#S>Up$~M7yzfY0!fIGthJs!|?vsNx=i?tPUGL4<_yI_D^TIKw}EBs{BP7 zZ}n9Fx(Sj^@|$ha|5F;VL0n_o|J*k z8F*fvs>)~%c^PuS3{Gth#2Gn#r`BEq(+1-PHS{J8D8sXDc@ec{QY`sXIdqNmP1CK# zzUl#L^L{Sv` z6F7p%cXu#_7S{c6m@_b#64Oy8xNdMWO?YtXSs2N5kW%6)DukVO}ixMs*+zvDZ&#q23HzMLQO+wB?ZIp5*jQEzur-^K-Ke69p3u5FfS z!HBW0=%}D)fMGKi1!uBNB{0Du)7$?JG5+LOK} zrtk58(0hXGAf*E%#i+kTQyj0fv-mdOq%n+P0~SeDQCE*9JxmI-#pEnwQ_*cTf?0f!-CbZfO`i|zQ-2Ug)hp&&^RJRz{ zhe^(+(sembC;h_jT+frUMcobAvRkk_sR1(YKaWbh;PU3Fx-wgMyT$8^>PXmRohIIB zxUQ0p#T89_4|E7bZ?X}hR%0_HrY0OS=_FkBcD2b@ngNo{6til1$?c3SqCA56HC?P= zs!*BMzU?Koa5SB5!X=;F^P(T4YtHUt19M##F2la)zp1CR6ZM7TiH1LTzu{VDPDCv= zS(Rk-`_R<=z;eF&^m+)dLzB)EbvLzD@*IG~)xbM?oh4!_Q*;bE#SsoYBnn_dKaJ%9 z2s|zO1Z(XIB+%}N?6XjjF_cntK<+0Y>EahXuahxhGtWbnUQPhXeb8yyw(p2sX7{(w zu?Ed)AE=wUN2>kqjYI-xqJIwpsLDL%+liH)#9A?>A#I^N4O<<}Npm?os+blZ7+i#l z?xdUeHT*lEt_>yqkJ?);1$$O2zb!lGN@V`+4ZXX0dF*ioLaAuG?1S6#X>hLpu2%zP z7AV9MX!0&BuV4|M1yme-4^JJ5jUSO8o}*ue&S(eB22#nN_pt3+a`fEg0ETFx<<6(c zd{61>6tz|v1rm@hNxdbLcBTHmkSu0SX|I!CPDEcmyFo+z&$`k`o=sY?wkP}KNn(4} z;lhhFmw%JYo|O1CCz>2O0Ot+fVI3crKg;Z&#F7coJoyt-8Ts9&z-wJMfp)jrv~uc3 zEvHUaU#PJth45{n#U3UsrVVN}STi`Sid_z-6&HVPCgnB@!|1(xzS+i7s}`ddC%Mhf z)^GBW{9U2Lq7C)<;@48<7PFH4qwmL1-qZ7R{ntZP38|sG+TE*#Bsf5IlGpx_@%x?6 z+N#q4ky<<<3yO+%#a0s!rmfU%5n7tQ)5%Jc;8!!CPHG&kX^0;musz!VEw6PdrTL_!splrLU9^MU`A=T(FR!lFJ)n+sNJ}XlRqZB?u$Tso= zp>zv+2CDhAA{WyK=?%%pD>52?=Gv>24KjlOFohJ15x&oly(TZii5c={gK-jOr!R;j zx#iy6OJ`Lc{kRzbKKSlnc76Yl`{sBh>E-!O`Q9HS#-Z0>eZe-07`y~^tmXCr;5ttG zb50Ybb&+634)kYL8_NX2}aZ9NmnP!acYd^I;W&!>BVc?)e@7z7KerR z;=1XIO&-@@AzZZ=PYRSJaT&lCz;MDv$JcQT%C!k=)sI1eRoNf6{}6XgyiJuOmEPm1 zE_xp%!kW_t4dH+Crm686b2=i`Li`J`LH%R+L>l?&)MQ+NysX*v zGCiD#gK_iZ5W>fA7sGulH4B4O`+6L%fom&(wR2Cpc>9F;X{T?M-JVxhTj)79qUS=r zUCWoFKWO(cmzGGWLo1DL67MRE>cvaNNaJtnnzWaH;-GNWijJO8!0`=mw!1thLbJG) z-+l3ALi2LDD5}^b!3Z-$h1IpX@6>@d=^}!3S<>5OPv*%=lHxHJiezdA0rl$MftnR&| zb_unh)Jwmb9-exH*Y|IbJ<~=%7T@3>J3@%NsWuy>o-9YMO54N6lBAXH8_>oT%Ivx> zFs_uy-$~}l4`PMyLc0u*%*N}5YX%tK|=lTK<#fqzM`AG2usVcg<;$C zhH1%YFxsO0;tlD*^TJ6s6XjFrVbHtt=FI^9#r53Y6O8@7lXu4w18$!VQ=$4lvf~jS z6J~b;A1F^))_k`oH9y=5YJ(3cWwB6*0JS_3@~!*fF=rEC53AqerdY9z-2E@W$)I%$ zFcqUD^S3(C98O10d6DxOk}+$PQ3Vx#Xu`#17Z-Y0cT#n@{8PE-5RO71uWeTG`e=o* z&FcoYEa1;dQ}~;CmCwo)g3PWYv#9`^b+{2{bNk{%Bu&FgO?JGZZEHGG9hUwC0Ef%w zJ~nNy_ejI`t*|}pQgnRd{AxKw!{vT26gcVm8|p_*5f+(=@mX%k!x5oBGO~c-N1{%t zi!JfAcRZb3BGW{TxBo-dUsvX^^GFl?CA5%W*)6xS zGOL#)%N8&W?QfTwoh|io?NoBA{d9ZHtlgA-4-6?xe528V9vkBKe||}@T9N^AkV9XyH}}_BLoQm5=-t3wU3+RMk4lc9w4d~JDn{_zH2>+h#fZr$BPHCq*V-N z;8_&*EsoNAx0Ch1swU}zS(=g8-cI*ix0$O5%;@??qY`zz)LbAH;q89%7lTmr{s^_k z;~p0j7H7I!Y!DulK#Tt=c`Rh7zC}8t(Y9YPZlMc)uzcqbwno5 zmkBa=Rz(?6wnmz;GC3o9w$ZeE{kp^_7tPU6&xv?GYIH^4USBQZ9G+BUSkTPf_FgBb zrN|F47wEGB#w~PW{KIkj_pNW}L~aih{SlXDorBmnlhVU5_hWj1HOy!zqR0!7UcL|O0cg{o`qKz~do`SX2E1~@lR)CW z%BG!R2eT#CzG&S;{9k<2#A?xw*~4=i(gwKkeZsrCn|Lxo4`_0m!Ukg$;~zYTYAInk zflzcjrJVYLf)&C|4=}mYNkNDby9XiZFOz0UESx@{Nb(N+;~87m=#_KyG1Jo=>kh&^ zGyE_AB-G<=##S|+Z4SN%L*BXXXE&wQ$R}>eLb8v!*pI!h_+XA2J)gp~V4J$Hd<&{t ztZC8J^=55&ZdN#DlerUCB(RP}Cm#8sB%*qlhayaOqtFf83-nAI^bbtq%X2hw1#lN& zslXPu?abZ?90_CeIkRJf!DC>dGJ3(+{1?Yym#I5S?1vb{6aD*5Fo0@6xkrlW6r-NhY4X_7&bV>qJdXVRxU5&5MR z`~Jb*DCwYH6gs6P{*RJlkMAxI_fIEU2firPu}eY!3_DkIGHf=%Y@<1HoO?<~(x^h6 zjRE-1yHxgXjDM}@Z@AHp5n zJESAvq6tp8XD#;h!T2ADj>R8SJ7%sF&(6;iF-CZ4bd1YJEIfFQCVEfX!RX`{MLwfW z8{~a@pSjDwk4q|AVm8`On#nTC>f8?#vbUV}qqK z;f^&SNf!_3a|=f#vpIv*Z2fql6O z=H#1}@H93=6?OzDy7)d=;unaH0iO(86&w{ct3M+|5{l;D=U}CpSz+TJIra?5Zj)3F zRiQ{+TdQfS7=9-!Dgr>|mQA7< zx2k^;LOC7xbGGvhqwgUjo(?eU&buYUfvRM$yWLIyKJGm9Q=jC*m>gBHUhW3B|q-o3SK6$B9f3A^YqA zeCXF{7N7k-YHW??@OQ3?$R#p>*G2d8$R{VI@T{i}kb{kQD_6@{7*}5El3J&>$@lCQ zLI*=@2^+T!n|7>aXgpE35}@;3rIg{{qf40VFTL+OKwIs3G?51gq&xDz)Fd%VGH^#O zNx|pye0j36VB<^?t1z3@Rs-W#_s4Pe2F3M ztt5^f@D|7c2@}U#k|2I^4&Il8hc=g^vq&fYx|jHA>yOG+xI2k=XZl?J-!&>U=iAV| zb#ZyTxKri|;TvQczDIt@YnuVNhd@hf%oH;A?wl@pd=4B^QVWsbX>=>58H z6Bn5LR8r5JG;7pQm;u(TYPS2}@1J^_U4h+cb4=JCLm^xU2(_V}^WA9Je$f0!aoO)` z9W^9`r!+CTDvbx`sk~91IFeayqi9U~Jn&LEoeqGuHuZPBA9gsG(tmlSixYXL(HZ6lIX!q7Y6^GAH-*>)spAXTE z@@rO_^MHWR)j$-U{72v0u-5saoch-fAIUGi`sqsqO6#M53@c|!={$51YA<*CYTTkf zenu7-;Ah(4Ppi~5#Mb`iLai^diFsRIZ)pkP;K}@rJs6gQ%syeb6WZW+-~|gt-&&u@ z?fJ;Ab=ajkBTmla?nw%aM*f0Hh*poVjtj{<`3hJS^Z3B_9Hb++itaywdKi> zzl#o_l3}cNYzE{yGHDl-X;`V~8uJJJI*9aK0zYD>%|J?&%hFFpHmTU-8MmxW??0C6 zj#^Bv(Z3wA3&;T+aF+*+u|L|uyjMJ|TL!}7WZ8=z57*@L9Tx#&oSh@)!aY#7js6+Z z&yVl@PqJ1OiPTTkye&^m9Qo?q3%lAE6=l2h+JRalqZJ#eYCCdqTG&zTiXgWF_ zoOf=CJuM-E5j_ySGXClrWWv?f$K6Zy4<@e^K0|6{0QZ`Z9?$N>WKNrPzc&ClcUU#J z9s}Z79iZRgvuw5+cUPV$uE$roRCYs^{C4nbi(3I}J@6nnh&^(>4+A*g=Z~eH{KQ{E zG7hv>Cfh=FQt2g%Ix7q1>|TCC1OsN8JAu1=5TS_Ly!_J^XWf{M`9GslYByH`nehGm zSLqV@lL$oyY4uKbk7hjAT@DzVxVa|`AFplZX;p;Q2XxkSRST?KWi!}PI6vI2LbpXke8w1lj(Js#6>B2MLGxeYJ$9nh6> z_G)oUc`XHd_^lg;2eP$)e<|cd4<;Xup`dM!@T`wq_4jbQeJLh^3pT2H8RA?8lDA-= z+TK_}CV;t@%U9`(ka=tty@hf%4$<)$Sq<9H>=g+ilhk=4s_J`emYhUl@S_B{wb>4$ zJb*>ccjL&=V);_LoC-t;v)r{QyN2l$;(K}ULIxBl=%9%#wOgm&Z0jUr)g8PgpI%3! zKrsH0`eiw;8awqYpYRTMf%;}1&^O3(O5IDJcHF<~gl>Ff>|f**1`t3*jxKb@c&9bO z2~XK@#fp6Z*A{_6(D5 z;;1)czrITL$dV`f7qVzIr&oLuqpEP;9%Y_)G6U7ZSK}0p>=>l@-c(d__OJ>5$Uq5r z30r+j$OR@a8oxE+qSl#VHO^N-An=1%Jdeeug@i9-qeF*YCTqP!{8k|yOC$a zx`kPN1BS(DhN24lxqge6;-{=MV?kEK?q65R2*6J4Rv|T{aY&#tTaTN09VLUO)@TGWv(eyt_e@|K~V|jE{O?}NX zR~v4>6qSzNv9=Lw_5YH@7-p1({W!sd(=ZB|`yVjyXZ}tb^o=z&V#yz}5B;#wBIZcX zdwe8alM_s@Bu8H3Fr5Oi>>0vAkH`%)^B?PnZkC@q;0%APx!REqOY!%kSv<6NN58Jf zAvgEI37eq+Y>lPf8ktSCE18Qa^BC2!$?m2)hXO-^IgLJPm+T8!dthqt~0k2VyV znL{ztO_VxClyg{>?uQ85=6)oGH5TtmSDRO`veJGemVWewqd-{24QD6sciroC80%g&U(1tO=f{@{r8#?ujE)jI{ck$=yKy!`i)35I z^N@Lr0x_z#<+M4+Pbgami@&gsmu>+3!)%#^I*;25pl+=y+Rke)m=W8SZCS3yRQS8X zZaKMWu@ybhAHu)i9S)@JCO^q|*dd2=GSS_U;z&>}xLdK0gVqj_VRE} zDa4(<4IXF&K#^Gm$fj_)-+uX+G5lD)fa?g<5f9?miv_kd?2a3KsUw6HR60Hnmmg|& zM~b<~7!ApIv;fh}Qm8ex9j0+Mzb7sVlscyL@YmzL@xXe+s}n5|zj53UK~ZO%)Zx(+ z$_*O}(cmL%O4`NKhZ<~vg%(9hk5{`-WK?$CIZQ(|pMVH{l{QG!m0vh4f6G3&%F@Bx zy@pW$13BVY`Xo=N<@R)SF$W zv&-SchFh$;Gu!qNQM(W=3U0Eq$tzF1v7pv$gpJgZ1LEPm*~Gfj!1*KZk+y%fZJ6`A zJa8aFtdoVtWy9<;UoPs+r)4ftHII_+7J1{bLh+AVj%B2!uN-2E?h3*nxNqSGeikg& zI>*g{x?|T*C1lI>Dssu*xoF&@zerS-zZxEzKAXa3)%>jW+j;kz60KY0;Oe{2+0!mb zz5k8h_vLT(WIUeI=p@uSim%MD5;&0YnRP#;O@hCRwgX_bx8Ph3ryy?IWsTF#I9t4e zXHw?ZrJDGQ9)TSj9A--zH)dIBCtR;b2AB3OTzK17$DdrKTFswGzID?!*iN}AhO@$X z>~;6ykqNonDoHf8ZWjUQTeI`@q4OQ@Bly~va)DMHOB4mZ8l$V@^afHLHosVFF^8*{q0>obYOItyBj25OR+D^(>`sn>(0vpQ6*Mh0ZTvuO@jaYo@RDTYtBEcLcv}guy5-`sm$;^;|3aqa*`<5AY?LWecv`r<&gCkucQ=b$v}W=KyxxVHd~I+8Vd@PY>Os)s2+fReNi3&f@}c{U33R(gOp;YCV$ zHUpCh_ZRpB9)w)$^O_miYDPdJaQx^d$d!>nbxWwEM<=gn#9OfxBJVo438{&r6k-Eb zg)>#5{Q)8L83R~36V_y38YMCeIqi#VbZa%MBoWy|RLD+i!j6!yhm%kw82=21c$h;c zC|kMN?(M0gP*QuKd|bWU>^ID#hdaNX1~eQ$=%`?VuG3*$Q!u$Qs2pK)B>N2K@6>lh zlc47h^wJ-R5~*f&&Rb3U)p)Xv89?=_>G3!$@Vu2PH1TlyNZSs1#)!o7ot$Iou2Q+Z z%!zAsz#(!9^c~m8vfvH8uTmygr0UqlpD4oJYS(0pQ5#GS_)nFm07uwbuQq+&9FiBA zNDoRK1P(jkOQTz3)?rb}>t&5YF|=xo>xuV_v3rS+&+p`eyw=<`dArwzOu{rNWzdMF z1&v*@InA3nf745zf_w*R$ydLMkyhO!=s7qTiE#qXluL~9R(12_){s)czw5@gdMh>y z&(-}AQ5lX)lq73iCd-Le&F@V|GFdl(OrbDZNLuLwM*kakp~D|a6jwY^i?bDD5AGBy z@5Jw+vs#W*Y=yXV|mV7oQYbNLY-udF|C%i-7&`#LA4e9!TLGEo=V!?b55U2 z3`8uNnqDxYH%};!5wts`-2kEJ%@4xWH)7tWq`YpY{DJuZ_PFZ)RcQ~G`N)?&F7t7! zxe8=j9P<&L2{H#;rLO02ACYQbNIrdahDIcP;FF=r;@pz|E{fknHP@RkPMcf-H#xGG zV2!!)H_GXfpc4xZw@(MU&Coq~DR>_VTJ>v4Y@f*HIDkvNdulAv<3>#$WG{gTE%whu z?+Euj?}Ru@UCzFPwqJ+9)t@$cj_y4KSr3Tc z-5+0^Y=2YTn#+0lClTeMc184390oRl{4Rd?+0fMWm`EvS}(LX=Ux7plz8}E47Pr8kXBwUb}bV71sjCL z=6AQQ9p=sjXqk~lFT@+L;KQhWu<;gEIN1I|QElc=&(3~f@x1?DX@ZVP%w-XOV=#zW z-`bc&SQS>?ncF1Z@+G<*8Sx}hxJ!y}=8o22Rz_ROy!+8aTL2{jd%g_Cq%p1f)nfB_iP&9<@mI%E9)!>~`t zuSGI7n&qraU$?3lQ!&39(rkckCT-i>{F*>d$^j!hO@;|#*)yG>ex(Obq~5E`55ErRIbSl|l70iK z&tEr2)0V1cn^B79#jY)DMIeC1+uTdWzV`Qc7F~DppvH^oK&71`D?{QCgAj*Vy}XVj zlfzI&<>v==?pqCz&zIAUU^60%RsInoXa(AE*jrO*s;Uuhwb5(JiuYcM7MK9L9FGXq z7`=0Zd%-GBc4(OXDioz3WzY)6CNGju+Ty38*3(#M+eeuR?J0(k#j%=fSnh7M3gXcb z1`jb+PaE_%v71yIS>xw|(RWx?loj5s}A7*t5T zt8RVUHBeji$nRM5!CXox&BNQf)F*()6u&e_B`Y+zBk^p2Tm~6BrI@2a7Pu(9DV^P=rYGT1#RE>Qqca6 z;OJrLpSGh>+ff1>;oPT9bd?9lXDi719r8JkO%$RunTo+1$9~Z!_=4_2#*LD90QWXX z{RqKm_CXBHr&rw9OXD=^{?u%ZuS)CMLT(L$`Ko?nMX z^7U_De>oA{X)n`h`TAtL|8hJVbdj{0t|3x;F}QGW6DJZ~@p6K5KXLz>^zekx7A^L4 zSeu>QAM1JX+3=jKe5Ns#ym|+U==(@fn-=n5V@5z3%XLh+@sPO5pT_Rn6Lvt#zOY&v z;c*wWRa-mP?p4xpx;QS#s-%#lvDIOgBkDnlFGJsO4 zoD16&gU4El{`g0b`1G02>2VO{p*?Wy(7DR6xcY&EfeHvoPNcvWwfA?yF`MyXX-4t7x=upT)Mz3)0L zl;KiI74*4P`=J!H(G6^g&mGrLuZv*Kh}&E zyI5|^Nx-wP4T;I9J?QT6=Sg6=`UUT(+1d%*_SA_;ikKB%iAIwem+c*Q7Nr@Z(mnbf zgfm-FhO&pCO&QV8rV?eg%xEf1@25TUAp(c>sDj>!eYg+!F7M^EX^n{bIcJmA}-AAyBkhL>XTPcdB2fMEH@XTXd2n8L@JU?G+rS;f@kx2E6KSD>enc3%^ zKJ>xmKy|r>DhrTw+fdL3Mhg4zie%1cA_~(S>8g+rt_qeNxT(I3T~fMOz1X$2bP-gx z2UDpu89kixMDr*yUVhOQ_367dB9qCXI7A-79V38+rmvBaZx3ck+_&C;o3>ILsCWT~ zbk108b^P#jWHm87Y`)yHnQt>Zw2Dz+JB5heUE{%5Qb`F?_|>;udVMdMw$+*)<|OlK zpOerT8A(Lo_5NXD+F;RPpo6z^ANc$H^hNk>SqUTTfqRSSLwr>#v4J^O=S`8^PnLNh z59+;bbzYa{ngpesV%Od$7Bez}xb{iCIi{MEgE-j)wa{|pO*p@CLN8=ejVr&V23IvM z-^=HL=0R}fRl<*}Qse`j$N+HuP7V+9M;g}CIzk*Izonj8$SV6Zg3n%X#8ja?ZE^yk zcLlYSIOTK4F|Yq{Py*I!2j*#Ogit3c@!lneE49MBE>VryBDXf5Kz zc;YIO37XdzP^U^=d3vkaTFSKL2n_w)ID`C=HXz zJ)LX5-=Iu%8h?Wbh91uc#Rbu&mDE5qbnMGvO&05%Z{-3Fb*_a9ap`^o3eg+pb%Z%I zIKE%!eHCqe4FBwv_!@d^cx8$}Yoil*{1?OY zfuJgu&M^Ub*5Y$KwK}4p`T&}>xi-Q+obl^fZwSv=D`B1Yw;UI*x+MNsFopyf-SbnN zbv9Q$?#@J`EqI+GAB$7YM&~zGiND9Pecu%YB6(H zrX_qKD*cAz9_~n|(UjWpg(KdofNf_UH5QUhyLl%gnqR4=bWi;Bin#v-T>RqLft~gM zxfJH)x$`=gWy+1T$Um_crwo4GfxutT*$%FZ1xo+KdO4cAi+XkQ@^tYi%!NbgUt9Ua zx_A9SI~p_&*Hb^n?$>gBJUP5IBd_n*m-!m%s(j%+t9svh{;hHRcm49hAnmxQ%>&$- z!6ymh8@|YTX076H_&mLBF|qcotq2zjy96C+)3pjIT;FtZy-yI3kQN_Go%58g8z`5* ztWlNjQgot1J+LA%%zFz&gG0e&RYEO$M$ivp{U;J>5$*){33>zrjv@75l~pP4wdKHQ z*d_1FQ4_oPc~Kq`^;T{An;oaq9xun1lKoo@2th4$ItRtNJ zAJx`4PN_WxC&c-%)fzHXKN%uEZJ*9bQ_Y=-?^f>e$3Dr4yLb&c$ThNu*^xY|>e6^nvcI(BM9Ad45TocrXn}uj7=9rU0TmTlsA7IS?7rZI#qGM+ZD z3>f=j2A>LIF)`zrwgZTaaktkFJ)a=!`}ph+F-dSzCFlFDcLF$+idg zF33}l7gQ>C^ytcAH46RB=fvugub3C3YKscthC?%@hUnZjpOhNXK^5H*zR&RDaj{)u=XVuz*oYo)TX;8 zadk&rXm0ww(i`3!DKW70ez~R>Ej=l<=?=V=N~U@d0)3->;!+8_q^w*OUBEbx8t{Fe zbvt}3Jo*{kU?9@Vx!EitC_{B!cZXC|%OuXD`mSwbUQ7>uIC%F}$P%l)OM1Th&tB(n zkuB1f!9ZF!^Gx>fFyVdcJBsjh2Zxh)ao=Ecp%nK|i=tbrqWwd0d5?7b5?uqLutN8` z$-LLvlLvYPfh_(Q_PR{nEQW8aW$zkyUba#aD`iHxdFZq)+2ljksU;Zo1^37@5IT`> zcFj%wC%zeNReq;EN+U>A!q_lWGk?e3Qu6&45AjYK?wLio{nB&^FGdoA;$w-XdvsedwCtZsd0wWWN1uo~R!2xZTsdo(I<`7Bv+p?tfvDJm^xddx+e z6e>?pP0LiK8uy}~XdV^2Irr-VQ&@0LHPL%C!aqFshJJLoH-8$Eo>HN)FjjLa48rZR zd`IDISkfLCU0aJICV}^h{Q$CpBs#5-W7Ibu=wnl^-GcFQ9J&MfZ`@K3E z?DkJAK(?jd4H3ImnnTLxA(zx#BPrpIfLu@JH(7%;tCf&GXVSpv$W6AOksbB8+H3iK z3AW4-n)oVY<8;uUgsP_gY`u&(MceJt$0oGKw zNf8%F-KSdReb!>ABe##_5Hi(bMxh{DH-9JA_xbhqkJet4L|l?#Kw%JBsHJ zs37xJUxab2(EL&6lm4C4GRJ`Cr|zf>d?F`>91QND^z>oUtePsHpzD|zJAKy*iv>qk zd7r%j3rgd{jCPGyeMb{IzGANP$G!=RLLP?Y+eCG#^GnM4n955Ya>+F<$$+Pi!y3%A zt>OG*F~mg8n$;b;`{XXQ8Xpewk(pql0%*Sc{nN=B5}>7tqXLf)BPoU>;{ZP$h>fvz z*hm?pBPk0XST(_6g7@Q36e(%WpN)Yuh*M3|%+Eq;9}!YFRx5o)eGXF#qg7#s3Wgjp zM;%%}hgI=)XOlpefavf(eW}TS%EbM%anfQ6OmJ*a;alI zqvamPxj=OHlW?c=8$0bE2FsU>NQb=*dg8l_2&RIcyJNX&vIPu!IT8<8^IodgD6|a2 zX9eqs%l9dyHh!likHVT`*#fjB7zs!POQ+utx_rZJ1USjBj~7_bEZ)|9Ce^mZml`Av z&9x>YBMR2ScATqsY=Jzx)}g;RhA7-yC#oO~_YZ4jrAjgGek_wio+vO6WEHD$=Pg%A z@Z@lu?3pkNf1Tx2wWqOgX%#wta$0WYcDw^p7|eH(>c?DSt0pET{rtWc0~NEL)W+$^ z?>WL_A`;BF1>KwK4hz$7;kNF!8OmW#!+)bXSaVy3yDDjZ!Ll&OH5%AT#k>CLf}#hP zj5V9u6g!WIMYlcVkSi981HR$&RVhjy=zUzxthLsxz+Geb8Z-0X0VX^^OXr^fMg`vP z73{v)>8m!`2Zq$e3U9%8{>QCkQ5=bJWCAqaZpR5@ZGy6APV|S!Z=#6$cOj}PPZ{Bb z;-!dOW|ta(OEvCSOJ?r38XUt!2DUa*yK&?q3`F431-_Kxu-`<|y-{~8rbdH)U?28m zr27?iMRdw=xT)?qN?{Yo{ZxV{M`m>O5!ZghH%6ksQNg-+ZgbP+?eNT+{1KjlPCzP< zB%SN6b8C0ny%%jW&qzeuEY-6=S*cNY2`ndA!m-F| zq;yLp*MEYb^fmGKVvZtm#d?Bc2Wwwii<1Enue>;nv%i z`Kh#6KbmoFO=q$=h>I{(d zfJ_BW%+zjrG&I^xX0@U^^^2^P#p#Xt`#|q9%v$k2BX!vHd0_*AXDKh!H{>Se35=WB=QoFh|DpejZO~MaH z30}!}!155sGRTog67>^K=b3t;#`HYWhk{iCR=hC_T^O&HhLhB#J14zeB!P(qvjfwa zjhUdSgOe_Qv|9X{-LcrgNkBV~gF|tvJ~g835{jBT05nFl>Fc-0Y|aFRctr zLGg9i6O(WLGZ~4geGnr@rYKLHBnqrjFH-7bM%Cn#a=nM_6)B`l2I0h2nHP|#H7U{S zHaQf%&1!&>47G;|3H-uYH6Bgi3op}aSK=7HIhnUd9}P~Vp0cg+y}%qwzvDeXCIXq3 z9r9qJ7YBic$qX_FiBxxN4)BcS{D74MX@Ok|G>gk9DwT}}BL)tBGxKtU1f0QqG4FDX&#C^F07f58&ASOA@3={7Tm~_d8 zaH5G2*P7PRZb7O*Enh#ky{SLaDON3(BbLp0gYc`*{W9;@6>PDBwW1)%DHil2d4MbM z?R*XL-A*fQ$rEHbOp_ELnAl>X9%C`&F{wm`f&aA)15WrBbE5n>p|{+K5^Kn~FKcdF z@A0bR`Rx!gMnc)U+K)n=Ar6I91c9zBzq5GRNhFgp$c#5uSl*`Twi}0vy*!>(I*J^{ zOqYy05SUzoE<)Rt{2gBZCPC_v+q|8x1vl)4V-)?);n~*ceI_^F-Jf?(7yekHrL(>HsX!Rc45@MxXrN|wx|P3D-549-Cwh$Txi z9ZJk_>3+-k!{DnAigm~acPEu$txY*c%&My)HrBZj;wyGujW%&tLA7y(=wBUAtVzJ3xe0LH=cl=5E zo7rv>ar4B6chtdh+H9<(S`~!Zu!RYn&Qf!#OtxVot|cqT_c5ph9mB3~ie>FfP1l=7;Tl7W6K z9l60DJK^t$PBoY-n6)ecD~&(f9#T-Ixtr={-nVS<5#{J)F>by3rFZ#el~r;C|8*5S z?PTX&=X}v}{}46_$!BsJHsx`k^0B?)HPrt*_~HkaiUU1NJ2=xQo7K7jS^Xe`Z9JkZ zkNUZY%3vJ=067%uJF*d^B&&!O{ecC`xG04HXg3JM4lYojRy@6Se` zJl7O|j*ipXx$ijj;`>l5nu%Qqm}!pR zsIi@3q1C(=|Bd0`?-9|Q|Bt=5imGc_!bXDycY?bF2<~pdg9azKySqCCch>|D8XSTX z+%0%;cXzmx>|`fyXq_HnpM>yEL1&REU96F+AS~A*`eLb?&Wi__cr-A(g~7WXKN*|6FEkq9?I_n6ICyaj*q^zdnNN#VE^=x0uBe9QIcVOXa;klLQTgOpcsANURef144|hK{ltO`4yd9|W}{X3&eNQRG{(oHyxxwB1{PYx(+1o_M0jti2hY z26H?mXfP%SP0C+qqQCeij_Q+@qq?`NW(s#kzUbL*Xe+wRWELwjG2+L4|dNo)d_(lbyqFLr~Mv&}jKK207?90o2 z;)RSI)3A{^M^TUV^jrH17kkgDSa1kP@u5bJ++njwnAFP;GVd@GEt})&Eoxa$dQK7N zOg5Ud4rp=*DOj1%qXcqaCB$XWogw8#Ub+@0(la7vZf{Ja+Aqs%<(UgCk0SObbz zd$s@P`oq5nkFs z5>tU)-RVnO(_?_??`h$?j%*|s62|L}eonJ1QA_^irI2Gv5CLdsA3Ca`*%I6ds<2zI ze4%?dO#2>M8s_M$B3b0RdgPJWKh<0o)I1?@#<%wKZqb?HB&KE&+Zx@*QCbvAV91OQ zbL9Ul60csbn*X6a=b^xvWq`QE&+x?4 z5mF5uh}6tiNR_=%e)~(!g+daX1yO@tPp@mz5Qdr)9@paR->hPgd8`s1(i}xBunatlrO;ZNXfvfihnEk zpP`g6{*R7?aa%;+PFGfgqmv$N^M}bGrTxF=gZ-of z#Hy3A5B{wL4B)1Ls1SNcLnqK{bXf~F#2UKFlVruq|9y`CNk<x;SuYg- zW5V6YXKn=aw^_ywxS#t5riMBn!WDCHa^kk|pq5b3T_12ar)l-H(cABZ)l|trGYD3I zm&xL|AT-VYq7A?xTX1W9`lN-<>Fn$A%`ioi*uE2pSFxnHP+hR&-t8;NK2Y64rKm(6 zDg|&_R{=|6Sqyq1`L@DKH-uS*zfUB;qGW^bpEvm%E)FWO1@(ARy=k(!BzDxe=5py5okR%inB;LQ zxy~j8BfME2{I}j0Ap2mmkX4Arzz0i(eLZmm=*z|$!HaX?29ow1 z5xZNUS!IZ`RU|Xs3K>;~y7e^$AU4u;lxSYj7Ms#$I-K1fN{e-M=d zlnowqx^lW&f#R%v8#*l(l#CM6_qgv!9G0sZtUNh93XT68<9|@^FCr3p7&(Tb&_zxz=Yr5|nlqZx0-s z}6we0i^rx1>@!<{VQbmFTvnP5%MO39pDt(83N0asXzdSAy^hNMv9}p_o~YiINBNES0_gp zll${IKNP^zKIcuLv=dj@y`i|ID^#ERJx6vdhJM-fNV?De%o6OON*6KRdDMTu8Hyd* zn9xZC6y88z`LT@QUz+J}`6Beb59ZlHnL-}LM)O+J#oL>Dzf0=!<^%y8!(klB3fp#| z+JQ8wa!E34+4`k*VA6-Ivl5EuAwGY?>JOZi0{5NkHLT61v%?dTvv=a^B}#WlvwX#X z7&I7rkl6#|KHCht%Yl@cKAI#Q8o209UZ8iL6eq5V;0d39O!OCE0bISidB&P2(e=Rm zI0R?csydS8e0f?O<`JS{s>0U&=-@~=&mJWyxV-&&2prYW5ELGASwR-%EgTxyZ`l81 zT6Hyo)~+i51=WwDDm-?EnP6#9Y(s?2(j20P{B_dF-{0#?k*vsiTfd?>N9yEf_^*rN zziq8P2QinFU~3-=pW@bwo~Bvle-DBF>EnMx2%sv&!BIYr?C^_0#jzm%Fa$qh;lB+9 zpnvH>#b19!@FyLWd?)tbhWLC9Z7q4r2v{cU{o-nc{{~?wfA15$0B=qtU5HpfZT$;# zPuL+<0e2|!?)D1pE5A3pET(ZR?dG_k}{vQI-kM16Dr4|)Isi2|}t0V@6+ zFvg_Gd1Jo+A@uw!g8ULWQ1hI?B+K7S%t{LXL!DfaWK982L^1>RgJf2q@Ne5I3yy*V z`VJqUw;21=maSFHr2k_-z?+*UIlvPj^#|w{Bdcff_Fux}^EJRbK?ESCCE1g6LRDS! z?eF&;KY4s4%-`{2;;$gUe!hZGEB@Owp`e1kL;t~e6qKhelg_3j{%b$hSRn8}*cz4d zblQtCrKKtU#}NJ%;C09k47Qd$;lQ%w;@j*$=~f4|rTQ1!38kNiFKt~|@y|JQ1^D}a zzP|z9YXL)aj=#brmH z&u35+e?+zq^VM{C4W)GdKGCT z?e|`Qg|Ps;X)l>McxU$G>6RMcRU3akgaTAKpp|ByU+ki7>;|a^!lV37ithD>ux7(i zqx~|`DY3rhS}yFc1O`6i|K`VJY+(bT>h+H^5*QgR8E)bK|3Ck4G%yoIX^!7Yuq6cd zs?#++@=}t7y|H$|MB4$E8uU>6bXc1SIMJ;j-O+Iu;N#5E|tYok+_~ zr{Ubw2$lT5^D$)8_WuQ7qBo(3{)z?sk0bs6SQJ0*1U5DA+fUogNeA$R(F98+@9|OP zt=iWq;=U(pK~l9liKCeni89Zm>7f=ByfLCM6QX5YJ$UuWb*2V@%@SeniBQPYx&{e zw9>}^G#sS4m``U1N{8%+Cs4}IkM&PS zAcP;3xb+#E#yzn;b6%LKI58nAWlPscVn6HN|2E;b+GPF)yISKx5kF}P|Cf(&P_$K!tgS(lV^SmxW8Ck3W z%DDGWZ|4VZmQ0bgM^a_@R>V}kHL0u@No|UBNyH%d?;EurB$k(NegvasW>%opXwaW% zC!VYeB*uE#%A)TZ{Ro6_2aS<}TnK>G&-sNw@1P4=iBqTeEDflO(- zYvC=@H$>45Q@QVLA*(iMXAESps$Lu*1b=O2g(}DhPBJ7q&Cfdh50JhIj?6z3;kG|FVXLH_Z%|Hj_*6 z;$Yg*A~>z`Kub($m zZ5Ybsh~vI0tG#Uh)LAG-9)TAfAm81e8;Y@lHmv1x9M)^={HW=5!J?7L;UI@DlSwLJ zBo!}sd(jh|uoZBJ(@6#fw z0k}0Od&KEk4U>*W%sZZ#I4~a>mIH1KrEk!&-EbqSa1eKg6L->-NksK`+*z5E9JW_c zO@VBlXJY+xElx6|hZDWA32c@VP=!f+LChsenhn*KwCDXVV$oTsRD*mK>4S#(oVghB zw*{`po{R53XCXN^l+4g}Xs9wb*Y$r~jiQL+OK`llb(am&yZwi!cZguWIN1;Wk4m->AD^tHxY2`{R1k=}M(i zkYIKQ8I9G@Spi+PvR|&+u;1-vZ+)eaMZ-h~+1k?Gwwo2S1EH65PjIEx@IendNzg;c z^w`(W*))%n`CFg~WT%GGiP)Vd;)5sCiS~0^9xPhyD)r~We!sM=inhw4Z=OAQu0*6+!E5Qs#9H#s5^m;nD(-vWdgywV z&1(&LV$bMJtnP`fc!_l&&DNGsKR+rc^UcR5Zd1=hiZ99_Y)PCYNo?}cIQeZHt38m9 z^4*axg|B3O$)eZ^6g%-u+DJKhZ8}U*RNO1nxHuS`R|4m!9u2zJ^BUwu-3kX>skK$F zD)R$yI*JnMs;6)hXw_TA(T8CPwu_yNP{mVe)6;E6itOAt3^Xw$3*XXn zxaQ#|N&1)_=0=HG`$oQ)gXUVj9O{PF{KIC-Ow8B$y$cp=KKPL+ai2*Z;=1v-jKfbG z@;vF=K=tm=3gv60k2Z!=+Qbp|=FH#@;91q{U1{7L5dmJ~BqoDIxxp)GfBU%KWV_Ys zdA)aEyY_{S&!+QYuLyB9o}2*HPzC8CaWI8boRist(CxJ&w8e#&RU?U_*(8H>fXUcH zsjDxOuXC_>71U?`tT%pLV;j-dnkyUExdQmx~?)hOx`vY>a%gaJ$~xJzVozyd!Lh%rs@!J+CpJ z*X}sb>8txZRw4!Kc72WLmt$6D4c5y@j3Co=*Wi9}PZxcr(_%$T>LY`Bf6wc>`09?= zGQ^!rzpn9iM!CXt7`b2XW!qPV>>SB}H!);ptZFsqMaq4%@%<%f9iI#&z=M2uB~89H zq~}P+!@MQcfKP~yz6T}nJTK`ie#}}VNvD9_9Jm+{adqiqx99cjX7xY=8A^WKom@C+ z4P|w^yH;LHY)o%YT+8TW-RhcfaT4JMLh>4NM zZQdB`X7aJvx_>tQ!LX9?tzNS!s)BiJ@~zNhgayoejY5#Zuja>nIIfp#&xVEX;c=o2hGk)FeQz63r_~(vR;}QF8Hq`}J6N!P z5pD)Fm1JTx9N~|NE-_Q?P11S}!w&y=0Xy^Aq-p(@pf&c6;<4O{H&ONRZoLepXqRL4 zu|^_GE+@>D?@GjQ=W3$mq1>ZwFj5t=nNz|@CSGUJ{bf(vw|HS?X?>p?@VZ$~!!H*otxkE-+rjr)mbGB-Q* zO!*evk6;dn6P`Vw%PXk}_v|B0D@CI0vMJOWSof!F?e`J{c>`4h>z6Gpz=c8NRS-&<^wr$D@E zMl0~>dbml~IPs>v#JL``bVtDsCKh;` zvAI+Cfe&`O?Ud|*>9$dwS(&x$9W4f38b$_kGpGNyXk4_X(QzaNm&rK!h|haiC4ZvQ zrdiPBzk~cym`K)NU9Ji#-8A)TJaj2!Bn`wE5&ZQZHIC8aZTP$d*g0RR7yHk@&B|g) zVJlW88{lkxZy4t_IzQuk%xcz0nhH*GMvY+&|JYUXPK`HcX-V6}Q3l7iq7QS970HI( z_Nv@`uCX_4zATZUYn30v#8Z>(CelLoHtIhS^G5fKGSu`D zc{FB9VglnWMT;{77Yl%>_|LwuXQ{g)f2{>=Uv{mfY>6~d+E5Zisi_R zwJ;3XJga+NW#!2(GD4@ET^d*VuAkF=LmG3pNVlW-aZ^*R);?N+`JT_{`<>%3%8l)4 z#f$J*wrc&JB=I*2pA63z1nu`@86B`cXk~xV8|So(%!mgrU9&T}MH~(}t2L;YIv}dC zF$}|^y$u`ptC+O>K0AL(?(f-Whnf$5@7pZf=!>OqgUV)<#(lQ%2|ne`!$&f*qWTs; zzgY2H*SBoBhKMmUj}aBaEMvo3#W}@ETpcSIb4E42(m=`BU%^=5^wHWFC12RAQvPUh zI%cxJAfzgm`5NNdAJu`LnR(9vre~9my*n^vVSMNl4RaTRwBL@yj(ET;MVd$+7}ioX z5{Pi0lonYX4%0AnD*7N|UW8|Ml${cZ>-ua8RX*=KF7J6`T@6Ct_??igt&_JWBDize z`)jv~6sSN3;^1?fv3<#DZMxvmBdjFwkpON8IC|k07>NmnBGCD1l1#NiADpC6G9ZMt z(N67ZOS>a)aQ{3FYg~WkV}S4Y@_`$d%J&N4Q-iJ?XjXMP-Fm;KBaW{S5}C%z;5drZ z1`KAZUqYza5Tg26OnW;bc^m>V=vLEZTF^?hTq+{5I9h~8#xgXx_OZHUE7p1~u7b*V zF&6Jh=hyL+Fhd1>baRW*BS5~b0v#I1u!SH8j0xx#dgXJX=E7pzUL^}w4lO`{O)*Gg z!x@Fm%~i}>YWkmI7GjFFTDon>R@gdFsy8&E(I|_<#J?iH6?AM(2*sdPS)GTr4f{OO zgKbzio?FSdDA@dwtoT^HTtQXbDlBNiWUO!b{+3t~w>!EE46*a?ktqi}av(kd(xJ6ugHI7Bg5u1O?M zb8~&^d;fXa)TeYbrt8fCtT1S}gdCp8gqqp=QpQ3Mf8lKcn) zj5KEWFrCOYd~9@VHTZoD~YzY%4Uu_|rs87rRvjwo#r{+SV+jbLpjVd}**UDVB$fYXbIS8{p23dQ5HCur2VvzN4OGY%8nUSf2BTDEnn)x4v#xyyda=>98kp-ug3YlU1rBIe`d+Bw`_*Co$wjwTk6$FO zr3<^Gd!m9$-l7i{(w?YK$8vYst?(PWq|sNj(-O-c8QP&&k#t!>Z{@A`PoTR4>F4E> zqd7$~ovaptp06*EOR=u!nk*{U=(7tgIg2L^z!S{!>JR6tH6Cz}3h&MZy;OHTQ@JMW zC4L?>PJ?T#J7tX2>6JbVn|^$leB3*EA6F#w`rf#=it=gyBraHiT8`=j&IpEY6OETpNbf$yaWa7>0M(6n1q$ zWmZ*pejeT}MxH&_(R{5AxY|rzRUSZJO#=~5zcfN|Ku~AQMFwZs{Nc|rDTNPY*lp<# zmgMo%RP!UWEK|d3jfC|nDS#|oWV&f<`*pYpa?D^d#_)sd zq~>{LTD-S!_g1lWpJQyUibhz6Bv>;qv)kj6%OR#EJD?^SIqK`f{~dA*h69-jM%n<^ zs<-5dj%NV~dtxM+0o)@pBh91AqL#L#$HXr(J2M~k-hfI8*@?BSF?n1&n2Rx!TR23F z{8>MA@dz%Ev`zjPhHLqnSkXNJy0t*PDR5ypYq*jwWS2CF*QC*dQV>C>&dH@NabeDT zmyXw6G*7Co(GnM5p}^RNQnB4Kq1(z2+i@SC_WG#R`{8GYx&1EjXXXW8oOpVCMz z8tn5*Y4*b2jwjn4MG8iyf!{Ee8dygX}PFgOzUYQcB3Cq{Z znD3Dwxp^Mi`yB}&YZqh(lF7@~ksnGI^U*JFx(YjZVKF_9$^tPd7YHikr)jvYK79OU z()xZOI9oJ_|M&ggc2|el5{*c5|C+3a zW=@6xzi>e?qkbSjWH3`~VyOSvh|BaGT|%yL#W%ly)1}S9bT!Aug`H%++%2LsjopJ* zvyo8KgIIB=3P^^YpSz`0W83Ktivf({n;!ze5GHdqfZgid@yu`{x#0zU_PSfRK!fKt zG1sa0g-kT(3t~#S8eRS5IqorK`uWO@0Rw}BWm6lQRlA76CSQCsyl*`oy*7=9bZIY} zBcet|WWSjAve7Opn2g(2R4N*?6}TiGv=87?9kDx~;~`dyDFMhQHh8o9RgXC{nfQ9Z zO4{`FOKS$Sc0#T7Xpj9ozxhfIBZBf-BvwAMQBS&y{WX>Ynd2arQ=+75W&g)*)RM|z zXCDgTz6skKg^5jg>83s8;nr#1O0e8A_g^p6mqbX8^sD!3pmb2sd3VgpP&SDuY|yDp z3-cUlKCe8Q>LYsHhP(>7c=cp_j82uJIdl8s69I2SZ2RN|4PA*#Z<%a$1H7H{Wn~$A zuZ)%#rN*JjOKNgs&B~Z^qi5r5Zanv1N(T_SEe(LE*D7#t&V9re`09| z_vnNN8~c_UJGL~k7bohaEAVk;X6HjMI@2R`&-#0 ztTuS~CjAxtYLZpA8mnrz7no(xGeQ!8qhSb-&3e}n{Mp*{%JxN>SpSA(>~?lgFWGny zY&>kse__@~@>Ose8O|VV1O+ObZE2E?=p^${KZ7ARsup zoLjUnv)S!#;w16TgUr=hL@L&L$O7I3&`~w4t&R{j$&*LeF-m9NeG{nZV%MeLY8!^p zwK1N~%dXnJ79!QrUyd@q=0_Z!-W)98&OX=-uZ*sKpjfAR=PS=2Qko_$Yu(B#&@ch9 z*_A7SVK5h1&1kSD3q4cub_F$?-y)gD34f-=Ezu-sGH*Aa_oSYwM6jcY?u|mhE1fQH z!wnH16a(}7ZMHD<%K9~;)uT3XegeH@D2M7d$U#M&WW)_TJ~GDChc8Yd2k(tg5v+UU z%gdMEYfC+dpHiRBPDd4RoUBz|UI6o&tLf;=U|*ybC$YiWG|Ecu$L4DzuVV6OA4<&z zG`*C+!Z^sM=6VD$&OYyr z=rR59wwa1{8tV^X(C93()d!j8L?-^KmIYqwM2R73KL-`EiSa0Be#?an_(*MG8tOGC z0xSsjMqf8^XQl`>Zs1rA<5QM#5Litn-z7;~ByY&p=o(zxTrD~(q$YzedR!N>2-z^+ zF?fEes|3ZBCj`2b464raDR-UwaxihIw)a$n*g5a6(t&4R$6*x$O>u4qdQOEq;2)iE z*+!4d7D;sSZf?geXwypgt&llEGg->81Ma}`QDm<@IuIJ`)s3D&rrul7gA{^t2 zN56`ns4EJPy&T=(U~u`0BoMjz%r#>TX-av;4G;NZG`P@1-492Uys!ciY_l;S)b1o_ z`fK4_MacqOd>PT*@i}={ue16mW@B59ruQE;4gG_hzr}x&>o|yS4~$qgT^1eJX^$U> zzJIZG%MLD{32$8g=%R%4SwjgG4Y$^a^*HYDq_nU8&nl6cjZ>W5bzj9QtNJ!31qU06 z#QF-76|YPtYy&6oz6<#ej3&pQdPu+DZ87!Vo_)zAak0$$fK{zkWl%eyQ7hFo@Qy;F zW)R?>VU&i!zuWXy!J`#9a9RI-Q>%D0v^3) z2hFh5mH|!WPo6F>y>Wc7p66@|yN`l2X`hZw>=th7k4INBI?TKyYGS#K=zVOyCK_Az z!H=Vo(M-kIHW;$yqBe1rQR~mnXp~el^ag~A6mbjaPfwQXV)GKtCpMLPJ&?^^uC611 zUMOvk!*0e_N~2~-k?k-x0`A%VJPQ zD6$Nb6}hqyvU`SciLKFRH0|p%Qo0#yD|0DY9~WrBsbW5T|M*=rv&LD~XVK@btvuI+ zug_WembT^~r+W;Am76rgDruKrNbPMG{|RtJnk&4ny7tKXzAnY}LBg{NWW2RL@L=~e ztgcQ}d1&hf906}dC^p;p*cARgVjn#YqQxKL)b_tV9Zp5oV| zLGFiic?V1{GH;)AzpHTX4g0h@B9Le#I%>Pjr!zoUUppo5n2cnQjW>=RMC97Mwt<&> zh*`YQ+|i`^cyPfAGX#6GM4Wfg8^@CBL90;eg|QREq$S{{k@t?qZQtR&6fT_04g4kc z`O3pU3RR@k&?B#&2-|mN-)n*-qZy z+5S71LaZ2y9&Gu5IN7$$a6FG`C@}88pvuTlLBBV8oKyFs{8vnaeaewT% zd4LlN1|dRzY&1T$0dfcR3^k4C>*X+#rn`0=xA0Ppeu6;Fypvd#$>D>+%B&6Umkm5U zCJJ`fv;r^-5$^ZpbDpEizNHddofn!wv#M{Fu0BILA9NU<+A))v9;|f4p)*Z$`04`7 zyc(DEdyD0VYRw-wr(8W)_U%esoin@Nq3@D4TLI<3{>%(=dg&;6Am zLh1tEPFid~dn!lIpiDZ-g+!~_pm3G)Y@#Qm6<&mpUDa04-Uy|Pj%t4Cqshh8d;SeT zRY(k}PbGzj$n-%Sb%KZt&3M~y4-}%LaVOJL0r7PzBV@95SJRna!Bil{T?YxOsQ|&h zNQ83+QexK$r#gEI2(K!q2Z|_wTQY-&Syq_)%s{g>P^Wm^5zE%N;(aOQ@syvOh`|P` z#1t#~IcG0j$2bvzTW=C3K4q={&Y<=Y@@*B4J_=t!5qG%@wP`XUyxPRb-<5^_#AnwX z{AIHU$aAjySDy1ztUcQ4r?9sY2Adqknqc#Iv02B z;9bXLiU3Oaut+9qx_}K-ja5lnbwbgM+yHB|6dCM>X4y~Gd>Ftz6BWB ztf2F*CC^E+s1(6}x+-_)@7Tcajar)#XH`ohf%~G6VhjrXdc*(E9YJ|}R|sbmr$~J_ zi@G&3Tp?HD2C|D))I|FK2GQ@C`N#PAC9q_m>}k_YaF~u)DeIkOMDU=F5ei=x{DRyc zqvc-n_kV)=itv;fk3c>N0oq z_CCUSkglRXDtl&(sfqvhvW=yNOOR?Fl>Gew4R*_3(F*f^3e{$Z0f$=g*w@d4XMK=> z^o4FqZcVEP2V25Do{I0BaCJI~=GZSgZR3~}G(*;l3HLv7f5^f7ueB;j@&vNiQnRAv zu%YV|bQrX_to9R}F$tFVmTAylCDuN@?6gy=iuf3c5s72eZo_5faX5PZ9l0## z_S*M;Onaoua^P!AA8DyumD-R`x&5gaR?C@uTdG3-G|x93>xH-9;QoWOKg-de-}Bco zeel&6PgLxpyPiOJ(`|Gy7m?8q0QgFhiFdrg02j(F(z0A$8n_%ohey;ob_G>TO5#7j zvJ&ClB?}%y49bp@)a$uDCT?b=YxT0z^=tJ2kX`CiNVb~jPE2N}oQT3f)%}8G5-73! z7qfp9TFs9^p|M?%IljDj>9{$FAOci9f{Slep$jugmL+iQE7pQ#WF1TJvJCl|$`LD1 z7DG4i$mKZ_k}mdl4}@H`D$*UOPqjI z)dUMHgG(MUV}2ICKPLb(6n(uDespN?xf0E?JazH z9*8Ic1mnbi?!wvwX++_N1{d>(0x+u9-cH*)c%>NlFELa|YY)wK=dstU~4}M4Yd+}n_ zP{tfT;Q>n6O6~6<$_5$rQp53yfAGcqwGq*BE~Xk%3gOv*Xq7+vjV-wnN3jTgXQ$D- za&PlAp*PlJGhERcw$+fBWlIMmqTT~|2^!M%+mZmPW|Slg;-^*>n}CYzQB+K@xaN{J zn0Gu=_cUDTrj!GHwP=sv-z3V`x4(T#O6Z>w|7W)5Pry7uU>*j_e?cL>aT^y3#nFys zL~3%bit#6|LRtrbqUpc2?RzYr)B#>8pfHyZI{n*6vB2#~+x+l)9b)DSyg6cX6=a)2 z^52w^1?NVRP5AXxYY*gB+D_i4=6X`|9YBv+ALqi1->@|4y*p^=Pv5?<>4G_%(9yxz<3rKf`UO1&K9>lG7Nn&4m0 zk5WJyvHdN6tq%NIcH5G3Nx@!tF%gNRlRijv^rpnW_Ras7y8jI`{`se(nt6tC%wI!V z!-e_n{?rUh&k3I()61*!uH~=}?|d^LtpQ}jIN|RfoD1;F|AUK^#Q(q;-~mux(9rx< z$jwFt`X|f)Ap`33s=<~M%#ll~{@PTlg9v}@pDQ3dCOZ19V`bcPVdJ?loe->!2l`L2fZG!m^HzyyE}W!{`@2GeQa+Hzib>~yLcS#qP(L~coR=cCXnp6CiaOW0jfhV5TSvs z0R_mTjhDcnBU54cR;*3}Ts0ehc{h~KMc$7=mx$Ue)8#gsl0_Ufjr88arTujWct}a} z4jBTtabaFs?0 z4R$XgqKYx@AC!HM?n7Q=7<-Am_K=)>HTvp1{|8Pa;#NMJ4S)CgdkJwx&iMB(7}Ku$ zMf`J-<{0nk{o#Q2;Hb#=4QU6ZZjomPcUvz^v>MP}@o=6+y@7buKaXXXD_)&9cAjCW zywyprC)0ype_48~zuFM~`OU@nC2y=8$D95SU_rVQnRkb1+w&ho)f?;?dck;XLKE>vh&M zT9=1c*y1?z4r~6gwrBSWYHzVpi?w)GI~JSGy>TF&aVl`tRfSzII_%CxA1&qNazce{ zRV`3OSd2f{>k*fnse2eEX?6O;V^7SC1ZW@XYkHlp;HvZua=u1)> z>eyGAK6#R%CGOXa>rdVEX3O24y4Ja08}eN;whbwd=Wj|SV_0a}!M0dlb7I{L9v-YK zW&$14@Ttrl&A*%2+;4PP8H}e4DyJ__3WXI~X-TCpbRO?-GHh#>xOV{8GCnSp84Sj- zD5v^&g&Ki7M`ZH4aAwF)t^u~qkN>9`{G6bI2Zc;?I$Y{yU=X5cd+?P5dgcZ3?d5(V z*e!z4k4C?V?8u{K{lSS1r<=oC(H9MhW(GrN^|qHyB{?dEy2?Feka`103^CUi19Fv? zgDPah1#lq;$<~C}9QyvuO<#ib2jY%lUZjyJ)1xxz?OYX}Ttj9`nD2Oy6fGwV^K|D? zf9SFi&I}7LO_VOcdK*OGC@8v1;UiQdqrO9ce|o~;6@53BW3k#F<_rb2K%${SFK*n3 ztI+SK25#mI&jDHnN|E8|r4!g3+$)RujPIn$d{@q>uoxLfM#3pt8In_7OoflMU zlK39EwP&uM^A*fwr3_`#XxN#FU!VvOuDv=aI;Ub48yyoyEj@dFIIYiWvBzAfo*y$| zrlf2rlZ2N_A^wh>_tu4wkhN@%n2ahe&UTbr4YX77;aos_XszCMWx97awl43f59d@E zP;Doq9_$veXIU^Po6LTMOj^j^ZO6`wz@_3gmi2Dk>sB0+N%xDH=zHqq zmalECOOGP%E1mnqPwOYOv!Qib_zHd?=pz~v?0^19BXTW_mD>|N~Wnzm=x7HTPK^y|g@5YR6^ciz`qoT$0c}5QM0C&;L4H;QbI=rUw-~6?272*}Iy) zuJzV2e~jB^<8hCg&B_zWNFymc&DW$~oc3#mqFC|fdcH}i<+|Mlm3;3$^weEH(AoCQ zWz{fbiK?aJ74*7UhzOIsIziVE+bnNONKlC*1mvDKw z-qlKdPO?{pVdH4NK=kOc+ET+kaHI_lc_#qoRc$h=G?92QUttp}iNz+|ik{w$?;IwN z7J?9zkMHvCyZZr6D69j!+eI4vtuOCutq);Uh|n>MKnvbCvc^_B9Cx+Ij?hp#D}E%- z?w9LMN^}hu&LQ~hD7}e0^|3 zX~Su}dh56m1YDLjdA#Zb^Io!?Np3qhno0`dulTumfzNXo*=Ig}e zwC$jY^gApm%_jZo!PULwO2(7K7IQ2S#0Ry9y91f0zGCqjQVSMBnP8P;pGMc`)J;*+knxm_7pG|0_fXzO;bv$J2TYmgfa}e{?1-1)BR7AdmOc@_+ z5=-g^eIHy3j|n~jXwEsy>*n=%f#@~}f=j_<;b8noq=F*9@b+DcdlMR+a+)@&#axv} z1U~O8ht$bfAW>u1$g52kB4?B*TnJZmzD2*+mrS@1Ee2?GL@8e%1`YuM?V+-A1WDWB zg{vdF)MmYzaI{!`Zm6HcDNlaHVfX6py7m2IbJRU+ke?gkY|Y9_q>mTO6;jYeR^-*C ziM2o^bA4n)YP8o1a4|gHP#Q1IjGfV`LB9_Vq`8|Xpz*OT?#snQp@%qT+t!(4^?)vG zC&L7fB3=0wtfk*6)WSZ$X(8pkndjOZ7RIB<8aV4L!Ko4Masum(AxBM!-J8WGp0O3C z$t>!aDOv#I;Q~F*-kn?1>7BV3#`!28b6Q@`xn;)ywV>r@g~qo(V*J@bQV?J;ZG{2QA=dqtc0dkDJYIX}MN)HsVyr z4ztU06u%_c4mUaVmo364*kpJ2&nlX&ha$E_*>8`FC!gdRJQQDBM(a2@!CEp$ z;d0QU*MGY4*hHylrW8Gh!wTC)9Szh%!DBJGhjCATN!^?KB7<*!8gTJ7h%_Jm1fZzm&;1HY@+>+qICAd357w&Eet_vr@ z3GTjdmxa5#yX$#)_xJAJUDaKEs`~uq4@DL8nfDym7{hcA5&jL*79^HVQ5^wiBdplt z)x=rJ(o?j2*LE&Hvv#9lD$e?jEI`|bmu=lO>Rt_+PG|Z21_bMY^Gsx1Ch-;hfDMSX zXPFTVsR~WIB|asPCt58no6f^%XEIR1L)9N2>7#>EiJD9jAxN%f1+%!@U0iV6*d8%Z zHtD<0SNN+bG3NxX#b!{K>)Bg?EMgtXaz<p2H6*C9WM~{8WTWu=Hyk(9t@9qI{Kv@;z60e;JrkgQqgcyOrd>R28b~Zv#F+{mj(?uDh^LN9OS5PGOlklLm$BNnvNX`5e=pp?Z}mfGrb*{ZL&S zqBMKzM!~1p_!}RG%6|l;z6~W*C+Xg32Yq%}tTTcv&(~e8T#nw1$b8jyb2^O6x#Fnm za@|8X{28qY5lE%d3bZeGp2R%)VenLMP-?MI`W@ikv8jk}wNJ5oDZ)(q%D@w%XPOdZ zW}`XbmUyFEPikLCLX^{=-b4i|d4giX7-SG{xw)3~}94I=oi zN;|Dr&P6zik9=I;(|sR@7`1b;B0(4!VnHH*W6Npv-@Z+Vr`4H{t?(n7cNzvpz3mX! zjHx2w2vImb#nJwbT^q=)JBL^Sh@h=2>GtUQ%ulN!y#)GrnSR}jpIM|?Rj%-RJB};9 zKv^b!P=jf1;6c7djhL|e2|aIN>b0}u>s4z`RHa)&80!=CV6)ewtgG_j!J|R^40SNT zg!F26eMFH`Y%yi@;wJ*-_g$sSG|HiqOqziv84XA@cElw@y44pDNsAnrrZ20s)nJ7Gwv2?%*`=W<*b zmSgvJ?4{{levo->PnIUKESzY-2t08|DARBIXB4!u7My?AQMd@#d(+9H48 zGt>yi=7cGFHk-nbjZ0B9VQB*N41{&2o=<2=OM?tE71K)z$eWx-kK=C?nnn2Ea)kG8 z600XHO~$YaG}Dzl`&1|yo}c~yee3oVuf=fI`tug2&iOGzlgo)r*`Mi->^eSQru~@o zY{nSdes0rJq0&%ZtDK`jO@uH`9~or*-Vm6z!#=qY5)5riMP?Y6K{dTrzSBwESeyyIA*pz;g* zGqnE;=5?(}02nxprBc2q-Rwdz>zH1E0Y z5t5I*m!R=%IFjwZq6F@DgKDL#8ns=XjC#{R4GioF??0;d5VZHG17%&Iixt+dJN9EfbNxokmWC+jFr-?{ zxd2bkwu_(#fk@m%&(`XvZ9Ok!=KgI_v4bOP<`Vo+*JPeQHJ%r@-UoKaEv?=cDfdT) zd-FR_=3Y0){b=HSm=8fbUk2*y&{>N`AW2*Zfevav#U>fR()n1)T0TVa9P z(kZM_SB%1OyC1q*16U)U!?G7T>J}O)S#P<`#tkK`L@{`;c7!X%ZGDL((%1q#z~8J; zIA7!G*PU)e`*tFKi^QMVFtQGFQG-RY`ssewWMS6bR-itA_z9^#ugSh*u zv8Fnk@enQW?k>AF(5$sZDFtX|tHlL3BtEBD>&e6mpsr>%j4+j2L z9@>@;8p361NAFLd$^>)ye?St~b{JCpP*dqqTExx<+(xYDmb1X23QU;#jxS8|)Z2el zOK+zGl}7B>S~Fdb4qH^|<$AG^fs&2$xt~PK-I`nusMV}Oc|;|4+L@k$!NrB2P-EG< zT6!Ys{OsNS77Wv{W@x4+3=t|@MSknT@A6R0SEe%mY_(LXMQg6M=Do5A__V?Sw;**+ zR+ek~ZZr@VYVaHK9>uNu2-=sl<~U|m@R}infJVMu`CYNN-2&3sq)~|IiUIn;LY2zj z$pZ4&m<>s;mvR}<*cgG}5tjj7G(q%{61CeXekYEV$4#gx1lqQb6dw~?B2S`>ov|d# z8p!?dj)Iutcnju<@q~}!b9qF@pJp8l0k1{?usHlf9 zQ76vI3QqSh)um7KEd7WrS>pNRmoxVBTrU)o?$Ed&QasUt4WHlZuZs%f-ThpRc{+G8 zrRFN)=ppz|(g0tCnGm>R(PW1$D{j!7Qxg6xrAaqb{g7%jjE>R%cfIu9bnWuj`Oh9( z>S++8kC1qF44H$f{>isf<9k02@oTj2yW~6}R-9Mg9?5Skw6>S_R4i;nE=e}7n1<~z zL}rWavlTA62OTze;ttN}a&J=CYp_X_iafu&$$l?V`OZ-^W13ESteCVV`x_dn`5YBt zMf5y(BO^=KCDUX(rn`nj#8Qx-*Y>Hd6FS2xo{9a54zjiqG!?l+@w~}P zbSAL03R}b1(KM3O{<1`Zt=ZW?arF$Tj2i%O9=*Ke(_QEG-EjHEbUehsxDV<4_9U~BsGX9kSrBT@+O=3<-FT~Gmv8Y zIJo^uaV~MR*$E~r+g|zeiIsTT!Zvk)%m)@`};RpwpL2Y0j z^4561qTMfdy;59(AW><;nS!?l)QBLb?V$0!%$i4<^v1KFnzhdIA5}^$9Nf_=|EzJv z$MIxXE(nlexYLJjdVg^q54Esj$t(_iy~!HLK*)QNk;rYnNw0sp3V)V>Zzg4YcrZN7 za#l`YtgVeaCvT};e6$BHAtp8+HyegDXOJ)g&D`9i-FRR1FpWhGWyIM)JZm{HNi`t+ zyF`jc%J8xBMxji(linz>9oa<%#3Rn{f?@P)kIo>0Kz{ux^aNzIsuLr{|25kmhV67) z9zx?>`|wF%?{IModo#mq3s^bPEA7X-OTAw%qOYKAW~Wyt@fd%EU?#g}BygKk%fv?m z!T!sZfb|p41>kkVYccgIqmbiouTK*@Up`%zVZCCq;x+1x?zwJsO&HBs7y^?F`FzFEY>Y^f=A-ik;M z`YUv~Er=y$p3No#9fFTqEP$skS3h-#U#K}x)LmvNN-BAvsDf+F8Sd5<>DZn!Q{ht) z@={OduN9F_Xr6(6#N&arUe>(I5CbS4-KP2WwAFXdaC32U05rx3LdA1Nze&=F(8WGw zG&zm!r7-%E1BKq=m@7x3uT=tf_(RW7?It@!bMXwGv4`pv`KQc>AqJjK_+)h!3> zv*ewNeZ(F?&rv)_$X%nCz6Jvbi@aNiXob6AFwG~*03isOTIx;Pj;Gl6SPKeyac zUHq0&nQiBK?M+c@zC93$u~N9q{-Etl3MYW^j9a z;^w0D$@2@-IW4FDq8OsHApW%YD50ll+(lPyZOiI)qv^xFsGD+xcateI*Y|1ob#Y$g%u4bld#F^~5Y zAX}J2c-=1_P4ZO#YS#p^yC-gjw`V?e>u5Y`F&_OQs(W0XKntiX+MQ=Cw4>pVRpt2H z_|t$1g5W9D5$j~1fT_{&#d2lCARhd>u8X^TJ2;UB)_7aVA_VtWq2kx;wpM8xF0Sc! zeG3Oi)k8QryC69KY}Ubd;ar$VIE1Jd`wR-L|uL zD1@81@|mZ8<6>&DJWuzU^K54$u}_GS16J>yb6qjLpYVJ-LIHc`RNBArRk@rxAy3Fq(B^XG7~-xVu|Ce z&=3Wy798SrWA@{IQ>G>B-(m!TnY8~vt*<5JZ?5iCaeiAgOm&p6cMY0>_k+F#6Z?1L8wQ0rQb3f=}O&k~tQw zZ&~)T)D~M9g1%yaya406+0_TKFre1N?pM6~-MKYfAR@f>TD8;L3bbnLU+SdsP@v~Q zoG3^Y64SZ7PEvM*A41v(F8nUL&CGp0VR=0}U4LC?Nop}1>Es#^TpeGp6Lw>Ij5b?0+}JP~ABF%=*O|Ll2qUuT4;HCe zcp-Z#fi>?eAwWH8e|O6QZaM6dCZ!?^gP410!;@qipJ2hRLpu3Us#f>Y6$MCgy9RH3 z;M1QHro^Ucra7`=$uf!li4CH>quAQOC8U%v7v#-2R-x|Mr5x-oFxGZjD`GCuaz~T! z(=rdW{`Afp)*9|*Mopb72xT?lP z;f|fPw_wDJl;u0mvNv;Res|@(ijl9KTOTRVbyy(V^qxCz2~1R1qJglpxJ9P-j<=tr zMQETQF?D9RjMr!8P91nPR za6YPwe@v`rc}VrZN!xo!(Mmdr>@5(UiS``{Ul~+)3zbg(&CM0wJD<|@s73Ku9a=N9 z@!o7St1MZiyyQio#pJ$PQ-;Xtd^SyLl2iD4-AiU7Utd2>aO@dB@RS>hgDpO3PP4E! z;O`vWD9IL#jo=NwUUa^Er{{&(7QSXvzRtCxhY-mq_NrRe{-W{9$L-)0Mf6TBEs{nj zEqFX5?bXFwq96Net)lUj(zoY3mcrfOtc*HAZTBq^ZkN@pP*VQIYQnbPAH*h$v}r17 z#spSag#th~kbJqo-=*uXw=^pV1}RLEOQW*axReh`EObI@5s{F}$&`p_G-2R2nDR1M z*FMe}IGy0;)MhH=8S2TbtYa0hJb<@<9?!CZ(Z4QK(#Cc;_xV7+1@!ibT9$0H9*&Y?byZB#vi%7V4D8tZxL znyp$b78cF4BW|52cSnkXlEocn3(qco?OI{I6Gpgx-02pi8=xSCG7iipMaZj_RB6o+ zI_i1h_|(Jvau~9ve&rIg)q9_h>>h%HFW`Azd+xSHf76Rcx!iq<n&;}>`x=Qs!+ z+im_Z%YYu%I6(D~QM_wEC=oXPk|26sm=-_EuklTMvAG7mE5J`{TetP-=D)$}4vnfN zx!S6wl~mT$DzW=^HgB8`KfBz|dDrWW1}*a_(>|+L$*WkQxeJR#a07p`nE7mKfHDW@ za3WAoYwzXY0=Z1zFpXBO&FTK%OLIDJDV708ybTczx&6)hTyFn=isGxu;OQ*w)f8VN zd|_?gKrT)ZVBaZ`sMaZ;`dW3nnc9j-O!Fg6bAvImn=jdSBN+`ECkVY8W6EXyd$|N2 zF8zXmRjqRQ;j&~>tBUdygOsssl0nIWkVDK;_#kh9>(luT3=s2hT61Jc*5B=^*HEQkJ+M$?q=g(4b+4>T3$R_v-0UtqwsjRb zRn?8{Zq-vjjvM=vD9jnsmpGD00CMGn4?5zv-|mt%&vs?ZQP)`ACZjP2l$TiF$733eeb|jQi6jPcn|4YAjwh=H#rqH#BbS zxmcdcBhWsYQH-PD?ix8lEQiza5)W)b+IafduRKKZl6`NgQqKt+Qt63*Q4xkq>)~%i zA76E%IyS^^rl?GWt+^(bP-~hI%6h}JST8<&XVwt`xY2BbZ6CQ$&=PA+eFxM*iilW( zE~o3K2a(qP?DriHmixNND)EJCh9~nP09(Pp=5j*ma&myXFK#PgupHx*9BwH-iCF;( zmL#y6(RSr@r+$5*UG7oFNa!qZcU!@MdZRG3L!UFl7M8#_ubIZz2Ov*y=jYdzO)t36}h%f@zU;7ZgiF6j23Sqqq~ zRdl?7KM=n4Pp+{tcG$8^=r+PWZTsWAv#K2Qsb@|*0j@&AlbFBfJT9M2Z^ii!%Cd#9 z<674SFXXL*9eCF$1$iL+Hxppgq5RQ?@~$`^f$Pl=()CIT7w={b7@yO6^&NP4ax7lD z$|K%(w!;Gz&~HBOt9{`%DM_U|m+!MP3E@3JK@$B6#b^6G-At}#!?srll!g^8G{59e zq0V~wK1B_4oof^R6E2$Iq}FC}I>q^$Lo%RL#+Vj;lZ5q$@{rWqwt0=7XPFJMiw+Mm z^a+;F>t5QY4+mSYT-^57qzAim>!0UIrVo7;^o>B>;R>e+^%gw!LPJiLEu&7st(~h_ zVgl-ZsESBU8PN#w728Ixs3v1CT`uJHnmEnI8!JS!YJrM-AdZ9In8n^Lqo5Q+bL>)@ z+`%30`<_Y_q0h~!dwb)#V)c**j|8N)M1!*WKUsDE)L6h58BDo29ep_p{waU=+*b-z{m?l`ryaL;unQkrc;r4~ zD(B9R7=Ana!Pz3+n^BS+eTnj#v#5TH1!){#JACZ53hlP~jb`(m{Dm#9!`qxxgFXY5 zGJOvbOUw)D9^wgAt!MsG%?Rnc)M$7D2Y16`Xh_pBFUCgb1e0G}x@^cB+?zCHH2>a; zR!IeWqeI?ZwKN-)ppMR*^PV1FC@}9#H)OI`@Av_63F)~_ud2!l#Abm`rCn`p@Q{m` zjGr~yADQhD=D*`b7A5lV)<6m-E-TN%Ram#w(~ia5FVC)~<&yrUlkNVdH$-m9pe|IN zefqDdbrE^=Af4qg>wi!jKp#7(>F|;D>PZ#k{L(Nv+v$H7SZ=4Y;!M8yT)<|==GP$_ z%vFl*$1g_Cl$lbyMe0@*Q6!ka5=Kbk(fYk{%g4>)4zxSBc0;nM&FPWg`J+$03(x)W zHrDU)xl7Pxp3hvIM(q^nS!FS(w`YM>sbljTAkL1%uk9zQq8`G=A_41whP@8)u0zghmu@^LU^-y8jnEpvBf%?@}e>tXTQ zj@AUl?NA7-vZ8|IY>zh_t7eO8=UU+uo32oi6G&5v+W&kdWj@C2mSAGUe8#t-Z3R>} zg{|o@_lI)XJe|E&nT;1Xa z1`;b<;Q5g~dJ26jl)|^YC1{yU2w|aMkZrm#3zU@9i8rN^ENXvJOy#lqo;FtJm_rPL zkjzyqFwtMkh!GjV#Chwh4rrp zdtxMsx+{|-A~77byM)~nz$oQJzP6nl1od12_ZM_d7GRq-wzZdjoV5Dq^BamZyPMkG zmnhY-Fv+EYwENnz$&{wm>(k-0U98x#B{`as62TjF^(ku#hyL*j)Bvld;!lRtci#A{ z>wZ{2D^Z zP=XuaD~PE5Z7tMpqZRPT>e*kOtF2TxxUWXaU+Ei#;M44EP;r$N@%D2$-#}h?Njag8 z>-_vG5F245wf-HJu&PPM838!-*^uBNB?Dk0y+J|dF{pc+ZWr|Bm)5eox{JRcFl)Iu znypDOFq!aNjx#dN|2E`=a6TH@Q}(hn0Qom)E>AWw9=)Zn>_a))IWL5%W0L z+kbJ0dj-m6paEz4i(gB~hO8Jc?LO?7w76t)1EpCTJ`z*p<6m^WpAS%nv>lFDd1ajM zTrMn^?(1l_bD?~*2$@Tr3vpoCS{OD2cVPz1banROOk?{AC2WU#HJQVud((5>Gv^)( zerPX~?hnJwJF>AJ?$+WT(!TIN8U@OT4d>IJklb8@cT51sx}~IRIM(+Uo3Z$*)0hCY znK{8ZvZx=Y0>;pit7#ol-o5b8zqfiEVAt@*kqw_o z%$|;$fHT?5o&wUPNd8RTrh~+sbnFIQ5swQRbsf(?VZ9wcKg+E$Ooq`<>!zuGsH^$qLvtA4d9q`1Ff zmThe;B#Wj{LDh>0!4v!JPGQFN1zp!GfbDzVG>m~K9G~e|w|@fsVyM!m?!5R&QlaTgtyOX^0l_CZw52MJd)}sq{wD>zlG5R%gy1WiT*%!h zs=ahq@2`v;r;sfvA-DN~J9;;ms_i)Pk6*FuI3cE?lL?31H}3(<_ZIrKPFLE5;js2P zyRCirD0fswfw757Y@cMDm(bvr!wF8yhp+AaJd1@FW3HW{FAhj`JW`gvU26X1R+#`A z9F7I-R_+4`)%=}%d2^t9o)Q$=Y-7T}@3x#H{2yXIU6no;)jv3%`+va6;(0xj6dH_{ zg5_Ett_bjXwlQhQZymT7w5~{Y7H&r9M~$~0NQhm@dl@+PED+GZ>~>*sD!5kx4YW>&WuTIK(4?*soHA)vi17J1lT; zxvrb0RH(7J{R}0(l}%ZribDjzyvR3WxY}2%R;qPhJP$ap>$kAxP)rqPIjV1=uKSW@ zrkmtC%7yLjVS4R<-<3^9$aVq(OyBZjvsTo4{l=c*^i_Mwa(5htE1i<(o}jP|c?Y|d z#rcejCjbp#<}q8UR{+t<0fBDvF;QS%kf>{=dQDozZMb+sAtsjOOvFB6WJZ z99Ue#ktJ23Sr_D91+I{=c_?^F!*k zb5Aeg6spI@4tIWsd!?IuKNuCq1vKiyrhhBVK=HkH$4*Sy$i?Ve_RPNH_OD?<;Vs|g z+_S(qGujYZrpv>ddc%yL)O4@6k%uR+Zvh}UDt$cE{65mM!Kl+W4jlHL`PQAJN@A|- zBsyM&ynmAs%ki<9BpJW{Mr6vk;C)GSs)l3z-?XjiRnkrTyDrvt%ExaVCVVsf-X*#7 zwFFdBLlcBbF(+LTNi20{O#k>4It2d_ulD`2FYdzUtVzhz0S)m$@If&#S-IgOCZVyt>r$G>sp=FI2qpnok}jnOLZ?@ytQ^Fscq2!R*WvSCS8!||qxLR4ClI~<-3@cY zt?E2xYM;@k+qDwQ+X2J`Kn5o?rc8jyoQj0+FW@d!ZTMbo{T8$oXe4?^$fr5$U`4Zk zA8Hk%&Uo;DnDM3hzZhT6{7I!nw;PW?GNDR}K!zA(j`H@Yq&W;Ne(CCName|4eL>dJ zg%0+7v>wy-PrW1`4KLzd7qq^giVwmkVOa4NG+i#?V4VGRJDR4!VHHKUmV-E*aW)MX z0si%MWaW6ALbg8{)?@_!g9-?OSjIZTV%zkwQI}+2PTe+oRo)^UNb?6haeNg|XE|xb z;Dg?s@)qD~-M#)xv(t3nL0|oNz9)o$HdA?zR@tnQ{@Hrg-|W3gAKPt&d?`a6xFv$k z53~iGTUG0h$m9qrsUE!VeC=x?39sC)m-XfZmOnStJ#-%US>G=u`;xc-JRNB@=~zj^ zn+jKv@sX0X?r^c3fXCJcgo6vtoRXurGZYbm*Z!SBi;*+`YKFs^5FO>^{$V0a{uu-hIF0F-^4q@MK6 z?vEjb;dmQxL2oIjXFuUdP!)d<-#osuHt?7Kt`kQQ6pobHV-OgMbNX-v9GVz1>`aDw z(MX?Y$DR_c&N>$pPL5Ct&KAU$`rGi4-;g}mC5_-8Kfbux2>DVD+`X69EuDw5xY;(2 z=r{)Sb`#9locj29VN=W$A$;C9;t!h|=N zZ%zk2Pv@qDid(R#jz1&hE)*9vSZMe$Iv7>IHFru(^kK3>i}UvG*ndRi&uZ#x#X+@u9n4KF5>ppwnr^8&xTq3dynUEc{U@1hUqd+j>SZ|^Yugc3gj znKGycf|QkLT%)4h5hM^mHAK9TzBP|#e{yE`^#xXO1#2`1N zw}e)45h&<6?M#^CX@bJ!b7-FYxx?q~1w(6yrwk$aX&V6)w@9ZMDjX5YZGAXeqUJ(%K zJr0eVgcXB>fj|NWb3}OllW8Rw`>K4sJ)Ke-OE`YYJCAg?0ncg_1DmC^he;n_yNxV~ z8X3RY(>b)}Uee}!Ix#fTiVGfxRcs0Io}q=y*+!9Mo8SdIy|MbR=&f+RYCh}7rDTOF zW1N{ry$am{%$*C9rAXid+2hyL&!7GS0!rR))z4^&_42sm(Q%M|iKLDE6B{73xAimCi zC!>f051Tec2Y6?h5i7-VK6@Ns&7Mz^qWGEVmyOOR(aVJ&WWA8aX`FlApfKceV3#xd zf9Gj~?Ci}L)Jlc75pIyyW8kHC=iGL3Ukbi8ixx22N}H>8NuJE zyO=z)o=aqoc+er8sq~EDd)huTB$U9n>i^(Q^+M`jD0hQFt=b>uUjMfP!G-fZMz5`# zv0Jm;RHj&jS}NUI?zenfs)Gl=v0U8|-)f5^=y}?4Xz;-LM>yaQ9yjYW5zV2cp%K*b z-T)nNl=TvnTqI4zk-!AW@s#>EnPNP;u3J^Tejh!IW_c)@58GK6s?|^8LCM;1aXwBJ z8?Naui>PD~x!8)&o||RlX?{8g2FFslz2A8p5aL+6QN$HDRn7Cr7t}?`?J|yaxfs4iUk=v37t`7M@NFKqok*epFB30G2vQrns@?!98StgCp( z#4Y$4cPVJmN|!Fl;klQb1At+=|NO#~4L z)~tqQED7?XNhoHOiNiTlalVA(U@K6s0m3-rr2i&I$@eWQ{I&%i%Ua4znnn|s1VTyG zCWMk_|f5` zWC-fuMljf78fBoBUhB4n%iws@NuxJ$cKu9u?zR6C$O}Lmw)rU+&H2&l=Ax;m4oIy`H$b!2M&6W{NQr?C(O&27 zJ=HL%1KE7Z8M@L&g^ZlMGtwOr&-5crR^>7ipy_7sfChOj$6Gz8yF-_>cZgvQoJe_=6{3H{KD_`dSr>FmM4R#eLmhPdX{dCOUpMDM#*6V$N zD~S5ky2-m-Q<5#!c|1aWJu(uc<3S5+zDW7Lb79X+*ld^>?Wo-G*mP&Kk^)3RuT|pX z{GX&#lW`l7^F$^82cBB{zk{d#mSxbY9MHe)yDORIu|JrY=-Og2IF*uNJ6|F@UEDgF zWoB$?N%)`z$&pDQU$bG|yUaIOXb<&jdf45LnXMwgF<(A6kZ=0#%Zh-pa@%H@M&S^E zw;m+61A+j}k=Kd9m)4!DXS@|;sY}F3k)Bc#0QfS=-G3m%1>`pq4wzcl*p%{JD{-3t zX61oxQ-dVfjG@xLF>m@4$PA2*hWIDFR|4DzN7Yfj+ID05jKP2ji{uq6}dVYDudzTSfjJ}w=f(O zu?$vzfhl6{Y`GKPN!^af{xsfXphDxT3;_Zh&tr!Iv5gN4Lyh$;nZMAzDDuq9RQX9@ zj7XlY_hwaq;cLy}O0;EI)X&}z(66U=bMYW0vB?#)7~~b}l(p&$6DsGaePJ`|H9ej+ z%!DQLz&57j1F8(&cVA264i5BD-( zZxX1$u%16AwpeqvQs;eZUMGf8$0*c|H<`2otD3-Mq4z~NmA%$#nxye9eC_E_sne8p z{#;@BeEbcEnZ1(`cj=b52lQr@RLje8?1D#8mYMGqyNA(|)WUo&&~+?AMwDwZnzcjm z-&?>AD4RYG^Oo97nPZ4>H+^ir01C_de8mo~6~^-NL?NO zu)dWC?0ti0X1Nwim0C#H^tx@l{w72YV9na-}#x-{$~Z=J)CfAI^;a> zxw|A%+do48Pra8r#=mhGHHM@NMMwRzQY#%tpT$2^o>FmY@0^kU%cf#vny>Y2H2-k; zI}`H7cpZ;oP1~SnFqvD{ST>c>iKAI@D2?A_qX&^vHhzHMCrx~40S~7d#co-IG8_Nh78O~~ z+j0jE3j=#*9MLL`YHO-PQQMsXS~xaic$+|(R7BF3QmNGMAZ>lGvY5T4O?IV`C7Z0K* zDdnrFvGi1I^)Vglz=L?Cxd6LWO7RC4n2~7m?@yvM=%`j5uL6)&5iq8@2WUAexSd;9 zl5WdB`Qh3YFM!}Vr=aVHRIMQe{b=LQR4U`YRH{IxxS%j+04p>PD|V1e_SMBtxURQ; zp0mN#t@!Qi!dhIpGQ{D;%&D5Z;S^bWqbq)5x*;kA0A^OjhPo_#S7|X7Eh!mVh$*fo z%EvwS1A_SYI$n*|=z{H5L9VQ-B5-M7q~Fu9m#nzbUR7PsZ{|AA&br|1SV&^rh} zA)%rkL2_Qidoc!8;*W}o${MG@X|fz7P`%(C%rnSgNZjA1ao0}5zv%d^V<5Vh@aWcu zhcPeI@s3@T(f?%>g{(|QMRlaiw*X1>bg#3J1LQv1h<$%ukVe9lITu9)`7XWzEFRq?B?}u z{RM*8?;1HIZU{I|OLfZ>2Gp~f`KdMS))yv~UI0TY+6N+_NJmyQQ(C#~Pw|4<$5|B__nt}C;z~6Cu|EI+PrWeS#U0fjeK%i;F|d-rS{#$OgnLeg za*t`8#VseuA`jHn#9;{~o3v4{UAN|aR4FP<0c^|FUu=Y-`X6By(XkjFOUi~FrH zaO5R%&Aw3uz-K^skq zI;3{utfnjF`y=n~b1oH;YqUPxydF@K27xTO_4Yl5Plx!Z!Lz8^o{X+BzZD%{Zf@5} zNc{hcjXLnK40-@r4x>+NC5%tS@lGP)nqJ4dL0{*U9S+}5Cd80@lTx8s6i$yayJp20 z8ywN&u|0sV3V;ACA;vnvR!>G|oTN%0s6>EqVA%Wbcp%&VYjV+|H z^~9HKJL$a~0^$B&MUZD54RwA+EZ+8nVIFYH-L2C*2Ze2YhbecJRal;J-0bY7`>W9f zp^;SAb7kaVb>zAJ7LcDicOk$32OVW#UMA2_L^8R&yWaJB2)G5Oj^f>A?I~1DO6vhb z;hEMb4^0GpGqWoh5ipTSFKe78N~%O^Mi6*fMs9HGTt4!u05N@3&ew&+@Qtmh?M=7I zWgLh;-P9a33?9B^#sXnJZ_=qG`?L&LWU@K^wlauFteL~PvZ+zWJ6(ha7OzEfdK-+o z-%m~p*nXIQ*{(DL_6o<1r?cN(kXOK`pfz+02At`dLKSj`Kh{-KaF0;|!V$SFcaU5-cIo<5a#)RIb z4qJ=6Uzw4jut+wIFLAvO(m%wf_nJt|hfy(&sNC>UUjhKelQn@D$+S{zUo~Pb@u?C` z>x)E*l{l;EY`L42HPS4b{5D${+!Z27HQ|pQg$NPbLfXh=h}-K)04Y_?ECSCp_|G7O z8n#FLwBZ7Ll;TO1=HzidRM;M#45j4BH&&*#v1gIt7K*0&L<%VV!0OB1NK%uAuJkMOCDXe80UQ?Iblp%K5>i`JU>ox)eZ#O|VBhtL49We%CZh&$- z=;xMxT6Mj-$5EK(dY=)p6^GXN`M(DR<6d>}lpUlIjd05uuxKED{I5Ftb@bpLMcQij z8i8YcI-vZt`lRWwDyD=V2)5)Okg*CRi8Apwn!m)rU)KAwew}Bg)GdX4RGB>`_4-qi z^+|LTIiiUcFX-7Om_xF+QB=-N31T)8`Gp%bw7{#;-LWkwCK>6n!buK$+8#9_V2=Xq z7lvYsGeSOlv+vBo(PAWC9ncmhi%E$^uMIf!l%sj**rA6vMu0+cc+nfU)1N_Uu}npW zHF`OoDkX|q;-m&w6S>+UDSbyk2>w)&pZ(EUWT|J05!y1@W?1i##rAx>AtP(keJQJg z&i-8n5x4vJsg%8h(Q(_K)`6eU1(KS*A&C;Xk*^B9;{kNL`vu3EM_c=&p+QpGRDJQ- zp^hkule4(&foamiL`2<&RzuoP#TZ^1U332b4RgXn>%jhogJ~1=>Xa}o*U@~JE%7wM z<2Eu{t{xmQ?xnF6l08kJXb=|g4yz(L>M>3BqLPO=to7S+n8^~l)A}fwNHv~_hHw8x z`9{Jc!EcX@f~_XBd2@&f%ji&a9N*2UaMt?^>^#|JIRh9HM|qazXs)a^1rMOM?=VBp zGE(@!3aa0ht7Ri$reu91*&`2Shi$g+IOwx|Fx+CP=8x6jv<;l||SMyA*62(|4dFHK7 z)0oaQUie*|&Cj;B9@hJR2+!MofbeX^3oWL7dORu7U3cCIeE?4K07X}BIp$bbFqj(k zF`SDdw8Fck<_X{QEVGcoh*l{B-Qwo5-RE?TMo^-DZd9ZL9iP{}kVORV4T^+lapJ4A zoy+5yH+Yv8N}^4wkO(_}vxDuBVrH1MDg30hjg<&Q-#jGx2-}6-GBfGJB#RMM~~f;lR$G?HH~E zKy00kzZvbc@gtxru}}0->^axFMFIV=3I1OA9pgg0t**zzmR4Ud;B=zDf64Eu+5-&% zy;k(2*lVKhT+P64stU4%SAye}6W&({h(-qRuvsJXu)u^>u`mh&uL0^rK}#P?>v(Xf za+);J4Q1wBIP_X_?SvIgIJlYg-X9`1HB` zElUXBZc%n9HNq{FuTF4DBBZSM_gPnm1e`Y>AVn$suYOFHn6)3$GMPQF?V1oW%iSX4 zmt|WNnr$Uj)^&YpWk(J88A9EM7jv{)6!N+$UG&}*!tf!8b}Jq0DH8ZePa!)ffaN5H zMwV#_^+r^+U3Pw{3$@y4AeuddA+152Z7EHJnmLV<8T#N$+{tK$;(ls1R`#w}UEdKN zAc`30B##1N{=XL`tC_2f;9w>?UXxv9I(^(8ou4#vFk*r<7!>hy;f&HNtf~jpcr`?2 z@b&uJk$cQZp~mfve7dy?;X_?tgnVO@@83>YYc5S%kFNgwT33H@2(OXmb;eDYP6f|m zN;66`-{=4;;I(e&W6F%gMeUTtt>MY~vUu4>lpuY&xBVrjL@MqrayPQk3loZRLQbZo zI;p(m5Jl!SOXecyfA}UX2>F&9 zbA}V*RU(+gQ*np8ASQ$ag~1sd-g1BSS{D#a_v}D<)3$CjUYLT9<6qYtwK0@D$An8R z)j)PeBl?KfdYUn6eBLw_!oM-1;@U3+E)8E{wB)=I6cR&-MVjhowdiyR1oMC{su$D6H-5`X$TIhlRP#DWqXF)DwjaaAoDkN zb1W6)>W)A3+IK)XxVXrre-3yqcTG7r3e5XL6hguWfhZq5!AE1AJ*hm^3f~E#W*B`0 z0R;=;u6%Iy(SM*yu5g3Br(inVi@i%FmAzV2RMhuvj}ITj7Kui;Cd9I3VtC|7QIuq2 zN3$di_7sbJz>WWls<(`bvTfJ5Z@Rldx;rJLL%Kl$1%?_zX(S~S3F(fJ7?76kbm%Sx zq+#fep&Q=Idp*y8t(Wh7V1CSXo#%dR$F_G1^kfqdWIXoU2226}YC>V$=+GnNWmCkW zr`t-4(hV^%>#_@O7&*6_*tzXNQTWJ&naronb*r}!K0p6B%prRV3t=4A#P=CF%87Fa zeIgSEr6bAt#w*tOI(P&T*#`Y{OJ*rbogXLTC{UpF@#oa!n4iz=R9XueATQ~lHUnAp zA-9Mh4A7en6L9AMUXrvbI1^+mm>#7*quEd;{+N1!d7? zq=r#D`|CF9f0CUwg3tR<{a+X3YkM9|*ZQFj58svr%l(kW2|2)n3mQX3uVhoG2j747 z=%d6E$Mweq%#OcU$NuQLy2cA6Mz{q$Ke3MV;N2xjV?~P&dznViI#>G_ZbVNDGR6J* zzi{L1w6X&jSDXeRNth;bKYJ+z9!gf2Qymxb$MbN@*sA9FQ6tQrx&<&FU*YbOvqolk09d zD_*h5SdC-?@3!W5oH5-3F@pBZ_@exWWzuZ^Ke3|ck_B}kd4P3j+4-5|Ieh2b6jDl0 z`fQ`7LL4|GR0xI&y!vM7WgvHQceaPIxE>Gv;TDKNW2Z(Q;Up2EBYB&OV(!TKWp~{9 z?QQ>wdRm}T(CLh4<=lA^SOB<*RrA~`PiLAq1=U82KXVmra?*NeL!7GPADFUJpSa50 z^lwr#1rFKYp3MnwOF^+Hm{aL|SLLB6Klzs+LK|c>?wRSw`yAu(UzL7VX(x%snCTPc z5+4qh{>+OZWQ=|<#Xq9T_3MG;D}C|tE!@R}eY{q~eo;o=T>W1bNQo&P6MQ zpRs@Ms*sYnfBI)(`zij@O&t)X@IQ-GbZPa(D#ebyHD01)Z5JM)?(BvoKD+#8C3Um` zSF&eTfj@`bp5F)sz+M%l&XlT(nwifJ&lEF6|FG74TS!OxnUYX7o2eNA(YHqF&w+^a zgldqV`mcUFe0O_0!*sG4&#ZIVW5mIm0BBp9Fj*o!{^PpT zBkjv0+>=8ffk#)LpWZIdu-3gPPUbRG4;I20OfXdWm$jt*5b&LCo&mbH0I+{;X~&Dr z9$p42GI(TZJ^ZFk41kC=MsR4Zb{jQPPmgRZqxn5lNuaxTb(hJAN z&NYh~qbczn?fY);DW{}*SKO{UEZxL3&kb9!(*1Kk1&}!h|6tkWszQPTYcYu*XgT~p zNbWLUvBT(A$2D+gigdCY?gClDYV4{09p~wyR*)gZKnXF6Q5caoN$j)-(iYm24GRn+ zC>QMZ6UveQ#T>f`|2~(rmCh-3{NKdui1igy#$i+9O24?QgT9Q`s)53;Ajv~|Z}4fw zN%L+uf{IkP$;~zq+acMiIdNidy^ARf@K<^3C$RM3=+rX*M&)VJSabujcnN zHam_EB}I>MjcH}v`q`I}xlBVX77-@^_On>}m8y{g&W$O)8q^g~l;J=M!bi zSED!03u0GnzgY?+x$WS)@rT8j?K|X!jLEj#KR$S5jy#j)&KY`?z5@f`B+{lE5imjO z4D_`G9I0**19^5V9pE*#0 zZvcRyHlWd5_j(KM#pzYQilg`m#z`Ce%cs~;S|>5!W1KGvugZzI4`4d@d6QVG2Qtv| z2X20Qb+FB3{Y`rwrXjie>p~`@r_ht;UkBVzMzJ)M&r1t=8ADC)PPi=3_;I?lK3Lc} zdsQC)+Lw{slLbVQ)xvmPO3L@*`uh2*zy`}6t4`ZZQ{B##Lh6_a*S=BNmC~p2gzD6~ zW12X%#-bMWp=3^*wSkC)mV=4WitPg#^LleKh?HMsnV9l3Y+!#lVfSe)ev9_o;JVA@ zUy#B|?2^VZlX}2fF`_hWG{UKBRZQlZgC9rHuuX3r?_ zRo5tw&4M#99WVIktF=7}Frj6-6KCAyGs&*cnEMU4cM`49(V%i=M`stIJLe<{Uc3)K z6k@Pzeu>Yyy0a*mJSX^Iq0V3tf_{zr_&}GO$LO;E~!e5mt?KiLFf4!$} z>tbcU22P*O`hpH`{%Vx(XWedcpW;N9MDZ4s$G9vSI$HT*A78#<5#JmCt>5DBz-gSH zql^CJexmRcLpA4p_O{-U>^D!f$Z9W)fBpmX@TTlqv|Qpe8IyW&UrC1JlPz-ZIr7yMLkpzv!IdYd$jv#2aT?1`G7FpJ{N4H4@kxzq0or z-*Hz?1yT9Po|`SiJ5Iq~vQ;CGZLQBoGOEzt1Nb zymWk6X)~3h*R{i+Ky^b4*ogmHh=;B%*TM6$la?8J;}F}ryRQfZ8NeqhNgR7ZT@%n) zW?u8g%XyjetY|b1@WbFRpB>2Q%>S31bGT0H6k^8rU>hY<{vLF5Pm#2|Nu*;dtxG=P zKCFAqPa-sC&}MgQy>Pucn=4vIfr%y@ZoUQ8|92)WF$QLz%^YFrllhLj>B zIZN^*{%e-r!kEj@1By&W5Nu9|66|dkXY3ewGi)UcP2e$RN$k#75w z+lz)E26iE|2aqg5U_rA1IuW!8q1({P`gL`fA*$Hk4M2zB)##V0cl{l*p@GWgi;B)5 z>|{FeL>YI0^s_wLXl`(M1iY!!5+RGFQj5YPuNm9b3i(@W1*dsTxHD!P8%l2A!XwHM zX}5-hx$6GK%397?FiV+gu5)b>&qFXPzpB+CZf8qCHz z<^_?LiL6I#KJITjbh!)T1)UihK18;!2nMX-O4J<-Bq4+{-xq%}ZuKvxFq8iG~au z_W@PL!}I{ZspcLT-9w zL_hQ$NeFxM~lcG;LwCf}(^x{C=tIg6ceJs=^Yu;YzK*f1U zvEPZ6Gq*7Y?lqO@mpSVB)(n#G#+O$@ z`9bpEyzTk!BSf$}kG?CtJFz?P;asln?vL&uqUI;P(0IRArMWQeB3y(`%%*N(Zw}&1 zh^*(r3^m23^(Y)6S~G6*cjFgr4G%^h)}XFtUjAT!)YFD9=bOa&mMxb-sRjIfQn@xT zgB4P1mskFa~fCsRz@yX9n@5>l5#Lu?N0GqVT+P&Wmw}l9_8mLP98I_V*VdXNAY)mAaZW+hbuc8Thd=`oL_r0%v3Ad}QcF-1nd`fPzj%IAJK z^t{>c=3O1GuP=QtS=QyNtBQSq4FCw?o?=$_Ujuk!5S{YZ~5A~EN+qZ|I<+4tVWl8AxKp2C; zh$lQzS;4DGVAUSblt3=F9TN+j*ISBpdYr3jP0#Gc#H8S{&lp?+PE++hP$I)@buxG% zwP5nc5xQ=9A^Pk@N`lqfh=w=~SJkTArK1^1D2%tV?XE_|ar#%L_=*D#_h|lLBK62l zrAcKNi$-?z*n&{=>6Xw1wcrUx(%3oNPIa(`^{+=&Xgv;6L*mp$!mI_2CTzhq_*5C` z$%W-zMqq8Xe_Ft8+vrW;1r)KW^W9ZRwqdmHaZctFF4I9d-zu-3fbH~2zjGo+Z=D)K z!oo1ou&rp2#n9fJ9?#ck5VLEsNX5jHs*`b9aPSC^a48a^zPxo9sgRnt+iQ~DAPkYi z4p2I#x9&}D-{zLPcWUJNfLfQIw zx#4Z`j@6BipM|8C>#!lOQ^*tcbMADX-w#&JO&vu7-7jHeF8zgvr~F=Mz8J{WtHDn* zdhJ-F6A~JDG(p5=PL)OZs*o;1Y#l7`YdiUPvCo09YYps^wSM6tic9b#G3p#gn*jee z*`P<9;PR_|A-#G%0C=nWWby4qz&jHem(YXxw#m#gi}|-RPD$?1hcqImIPq%+^&C-atJhRKWoorFg+yy~lOJ46T3-#l=H}+niHKy)`K8IXP~AE%u2=Aqj{5z9 z%7`rNjNmxS;-h;Ry9Q*0Lu~Fc`n}j@&NIz)@m$XF%a{e08ZU;_gW+hM&g>#RwkL5l zVpcW#WDTd5##Bw+b6qt9w1-gZ@5u9zdy#}c8Q8FB-OMfbaV^$dIjuC#^4zq)pGKNbOs+aa zvjKbVm#$d)XkF4yq4qY}ydoKx8;)0!&&efyL2r+UTQKHpq_q|aFLU-m}$_iuxz zs(y>cVs38-v=h=TwEG=?OXdT;2>$8J;XXtF3+r@r<&p4BRnJ5BgY4>T^D z6&RO%pE&ztK5;inG4efd_e8%vcMm62o5#MRj!Y_$apzj#GRP3?CV@lqI#Z{i7lsfILOW;+XZ@-=%=KmV zv5Pna(kqpMMl<6`40G-X6*AGc4F4t?!#%(%hG_q|@4V^;KI(B#$o|EaZ{nmAm`#oK z*{bUYd(ilcfGymu9vn2nQ&cnzH9|zbBv&`B@sXq5Kq8Xlc1iRvr<7X1ygecX-xk1i z4PZ2b>Cf{7IrT8;_=~E>;j%?cxxV53`6jKlwqoB}&)5>zqqCh_yiexp(ToDNe zZOi*ckII9^77cfbA*WNU^OwD;WPR>AVkYL+wYZ zM&3jxYl%o=CFdIPc@yXHMqumtX|dk_>xTP0K=jr4kMh`=Zpiegu94|>wQ+Ogo%YM1AIT}LDhe0xz{J8YA zPbGlTES1-oZ6t4ygh+Dd^U$kOd9CiJO^l~-5w>B?0~if5j9S!*L9*cpP%GcU=8bqZ z`&@v^X>G2JffQO(?<++fe$XX3>UV?Sr=K^~h)2|@w2QU(^bazReN)3-S)Od;K)pbL z`Y6|y62`-CL4PBtzr@)#A)PNQ@S$Z6ODpZ~iR5T1!l29g1W>!OWTdT1V@t^lFPXs} zJJ2NH65gNpUnHwwRSOT8Nnzu(VG0Cno!7kkt?p4r&l4ysH811T^AHZNLlOhk!Svk@t<92_OBLLr)P1 zS7;OYXYpUUNCG$ggIg&WFl`0g?ka1T>#af({^TrhhCApRd$AtYheI|EWpi0=>zYLI zKeQkrWx*|C^s1AvwGUU!?QZk&TLq55-)J}1W=m|1)edf*yJPv|K$Ah2Qzn+nA0)Ma zuY!?YEUgUR#kZLx{S*x>9T8&(*T&TTitf*Y)1Gg0Q6oFp2va|w)=cDOwaW1sqpNXAF*m?Swe9`NBc;Q~t9QTFYwODCA)4GqX&hX%g4H@QKv@Dq? zp~FZ6CD1w0T@}9$QnXo$yy!9@T7vt%)-TdC4A+ss+xz&hQIc=`Up?zV26MhQL)tjOI}3{3p%pC->7rfW(_dGvL&_YrM% z4)@7!-hoF{B+Gz^kvTr-aoKL7-rJ~law$havD%Bx*Ad2&j8YkdXvtj9LTT3#Q%6_E z@8TQ50R40^ci5S4$?(5i-cAHGqMSwrnKv&{j&kZ8f2nhD7?qm6sxWPSXYS4M7VCm2 zL2h%clq}_)VInB#_SLfAs?+LE9E*PRMvZLytNafK%krrspm)i1wiMLdyAjsxYG>1N z35N~Q!=XKkoJ~DoQ$C{*C#h@fzF1)%PTjg3fhU4Hf0ow9Rvjw=hHtjZWxhf|UOoHN z(Oq{m=z)T|#m2czMMun@YUHqafPCHl%}!T9R^ZmmH1rI6wLyPLy(?-J-5*h2X4<6B zHB-*hR#*8UMPJBkMV#vXgwVs+2Do|{wV-K`lbWyeMs+I_Q1bQ;kv*48f>awcI3BsL z7WkSLSH5YRH?>g7ZPm|Gw>nI3HYEaAv47*$5%p;7Y^28pesv)4t{G>;tJdeN^;1wn zP3R_2UlQiMhKNw;Om25wiCw8S7B&ANhRBoOGE_`8H3_X$)YYrDp2W8d1gyRSMRw^G z?B7Q_(WGvC+)O7(2lIVDp0lT=i)&@SWAbMWmW5v*-l2ejyxXj1)iS?U1G6$k7<}q&4b{a8hCqGT+6B$VN2> zUCFHSUJxxdxV-HdxGm)E8*Os}yEZ#wHA7Geyl>XP5FszFWMR9dOflY%T#suc6z1`R z8pLyrm&Y;;i!lxQuFXowF;rX!#fGasm{Ycz{pRSmC{$_O!L>GttbT4duu6vscgtlP7{a5{vtv)Abmfzp=hkjP%mXQv+g-m{Ma2s&i zd&}}&#)?04@jmK9c1^Ea^Wtt(-YHo%T=~g45L~cyrh{|*kEN_#z(AMlvWI*l^}o^Y zzjKkV&pzi2d^vX7sh45hLPE9v@ZT%|zZmN2eAAjEr%Egvan3jOR>*vWqpKc6EVClw z>r#A}z-fG%TgU26z83T~+sM6%?W3?Fxic`!1lIBdSNVUQC6>+C>dQP_8thKx~)23)%ct- z-MH_hZJ{bO4{gT7H6bnbKgt#_;^K??xtGA5w>NZ~N;&Q)DW7xK!v-?{)M_nO1It$6 zzk3#Crl$nSge5kYLnf!Uca@N=+hD>ClWAMt49JU<G`YbQr$>3y_Va=#%;P`{ zYSOj$Q>B1A%CxpdEsmXoGL^?y=*!aaS&;CDY%k+BhlI|9wO)-i{=<~v*Ea=cDIvSV zg;+6axFEDh()i}5v6S|ikp#6`cI4HjaAh&v6P5;f6e1F){5Xgq2WhdHSEl!42 z21fa@@o0sKY(1=rO25&dm@1X@x=gRY`Viz+bboUNB^eet?klG!qNDY$O^;%hY@auI zaqjFgH2%S-5&3~UZmE#4!|#+16&syL63zD%;fc=8Kd&P@Gi^H7CqSJx%vzr@O}y24 zTO$?SZA9hX8p?C}Td63fxXJ!WL9$Mb-8cUgv*p4`eox7IQK*8+RUKG;8n z;qO~lL1dgAbLflt@r4Mxkg83fj-tT#jK00Mr132ZRqyA@3rZw)@&qyq)fU*ctTbB= z!_=wP2E6P{vw`489hVnowHb*8YRv_>E}L}ltoNJ_4khjTKXK`&Xxw+gFP+MC~WJ=LKu~oYA3lF5W6XhcU_cqI(48!Jr($?!Xb5 zt~&)Qe|%-+t=JW76n{Xz|cmI;^V|Wha7%7sufA z_jN{)YUNU01L6Kyr3~{l_}l9&sxg@fAoJTwe0f@Jnt#_zgw*yFyt%fBUb(&Q-W-Bm zZmT0kX5ROv$WoTiS>Ewo)?>Wp!m)cy4vL4X#Tr|cgnfHD)uM6 z$kiZRq2n)uJ3Hk8nU0svyg89$w%n#Vfd1>4a`D1^o^Uv!mBBX=FWDxWdYS8_wunP+ z4A^y|CK0qo>@k7M_lFVB6jfC1zxsNvcQv2M+z1{5dDL|QprY<{(eB~lC6(c$n4cRx zvHznID$6`qV#iJ|waBCQ25?4o4Q~hkVzbds=g38P-9#|64HhHig{mAi&w|sJP~{mk zfs625Xyarvvjd#70MUcfc_5Y0Y*3&D_S$1!ukJFo-KVvkI5@VObP-caWG7l?v?$Ue zw7fBb6c1*8Lg4G>fUZU_L#t#mP;*t(%6E)xk9D2s7^# zq)NK!BR9+X?=(QJj0xdCS9huxqbYmJ5?IFeEGh!6*Xj*{l78p}Y4VjijQc`(I;~D^ zQa$nmXLN3cNLa$Qwqa^}g+S1f)EN%agNi?M5Z|NXyytsw_L@kc(a1mCL*KP@^-rm@)nhtH@3i~qS82zQsCQP zWRtef+KN&)A@ZxRW=fb@W8!fC<9rD3yrgtKgJL}dYtKWmg2vAKbaQ;2Ps9w_#t|bz2RA5lDC3dvZ9lx zX<{HZhsB|{f(LQqYTV|qbW$#p0W}!^1Ej_-S-qZnISo9c@Si$$sdUUbzDz4Iowfcy z^CifgzlCUGU6OSO{Mo`!=@Q>uyi+T^E&rcSr=N&-x@|Fm)1)FFVbufo+akUgCZAg0I7yK}AC@`Syg(*8RgHKBu1~r+$C^ z&O<>rS)3Sml=P_cDuwrULOMK&aONo=@9^u(mQ<2v({Tl|4EWn;0YmHxZM8J1VG z=@uC;^wV8@2MKxZiNbMr;`@aEO>w_`;v#fxa*?XmTb4MF4N1dEQwYUT(;Z@vpnA#| z{E*ZK90q2{pZ94$0+L)!=F{vDMGX92kvGZr_o3lIMvvV0&%bg&j)_r@SytW?x5Xdv zflKCkgpdz(+BT)96Mr&xfiPD>{i;*90-(RL%Kw2AZSzworQj?;4r$5Y-Ot9Fqdn#i ztvZHfT*P89SYf-+D?PBdviNJOH#UzVTjf1b4l+ZP|Lx*d3YE)z%@E?>*5Wtr@UfJC zDM;pr-=f#`T$EkGQ}!cqyQ_bKYl0cOH&yNS23>CTJuWkWFBN+WmCI6{R+zuEAUpx zrj1dXN2-ZVY;zq$=1j?}yO(}A=I}jc?FMit&4mhc*sVD^^VmiJ&>=k65?IPPz7-@k ze9b6J)%XzAt}S4@Vl2t2eg(7|go^FN9m&2y50X*}ym_RsG>;yTN*#;14n?`;tqr;g z+`SqTOBHjTcWX0*2c7)<+j3Q!3?!r}zfteM-uPEWg#0@{BTh%iNcIO%#R{QBA>HsNGxxazo_(iu^E z3$Iq}A2D@p1;1%`|EWnWc$1n~J!4}ykEiNG&3dr6{Aa{Atlh?}?ZW-$06f?3(hlr4C;{b>dtg;-O5cHD@!w8OhlIE7 zHybn(bpWJ1@mYtOuxiQkGUo8!<&(q%*V{!;ctxsH)qCQrmjy14MX!y;#2xP+NG@ert- zaAxTx$A7V3loKpqWVJ?ye8k>h0$QJ)o_fc3-8w@qqyJ26j#f5+TQd!AM&3C@s< z_tV2sS)jz4=3%w8*Ji!M>dp4I;1MN<<(B5m7ryE;{tt>J`b|Fs9`|+|sl8uMSMHzL%mJhOrF4{uTv~<-dj)#aD7ky|MT`kb4NQJm*Jl#5T zQL}NN2a4zXq?RU=A>iA+yvosWz0-Qk)HqMu zN}|zaPP=AT77DwA9AiR?lZ~l3csFyB5h4|{NXm*`~_32v3itc zyWh4X**Gro01vP$0%Iw393A%be-HTj1x0v_o(BB)C&s1?Ao4(vF zZNrB*HPYsELR-I0JBto?nw_%}PTM}b&>3V_K=*bNT*NXk% zVdT3K{w3tX;wP1W-4OhACd7fwsnF$5#KY2B! z10qeE=U{GoZbofeyuR6ii#zyrXK4`8};REQ(zs0PRc6kCTjQs0Oc{KZ^ z2eRbVT7(Bso3PG}oE z@KFn>M&OyLFr&b}d_52SdDpC`oLp|KT=*cfbg#6Li$bJ8NJbJv_ueLuEnz4Es`BU< z(xbv9mMRazCpOQaG?@nlB7(NI3Enn*)Q?Q#b4Id|5vp>FrBgrN)o|#4ud*3W(ZbK+oizvAS(CeZu?Gio z^Fo3pD4=BoWVnV*1;g>LeSU2I1KaGNczWFcv z?wXNQ1~hsBu$`p?yzGmqc#u5uMV8z{FU!_r%`M;X>l{}kh=gb1x#NrmHYultD^$d* zW;IjLGwg(d7fXIwx2^pv7}(Qdaq6qnUk0iHpAn!%Zl|T1L6d=Jud?_?Tqa z3u2C9zngyPeDRX--fDBr1+^&%4g!*okz1!!o3U)Qmdjlx6Av&Mwr#ZQ+5px8HZE-RT^*Y;q( zt9$?1r7^^`X$b+ ze`$94+5U(A2@m}%FS|1%D)Yd*q(!GD^^%$ig!%lwD~O@lOEqWx*~`_Pn|L4ony_fwlR z_K>ELbdhcs-`RFR;^%mKFI@Q{5lAIW1o;&tY>C~2XoDxB(;OyfFflHNsgi8f>(O5o zoRy9byy92@fHYtx|BB=hSjJ_iQ42=JN)HhProQBt+bK9kRP7JkgoTd6Txfs^T@$pb z^ZJw$MkXfTbzS&`D3*4pBVkczu*B-xxe)x^$3Hyk*8rqM*rdb!lYtQ1J!_o~`peG( zRsFTCC63xjGWB)4zp1Tncy5o~Nu;s(u3hq?L!$mDBYb5_H^Dy?BdPTrVo zM-KK}n|fk|QJ-eIU95brH7E0vS;OlIJ8iq~`Mo}i)ec2&`)U6(c<%TT)p2ByRvWIN zd(rf!JE!SQRjYZ0Yr=b@3V>yjd=b{xM!@)DQ}Y{73BZXgS5220DJHV&*e;IorwxNu zN$dLY=-L04NmRil`e8430YDn&WT|0J&3V8@K+;hW zJdjDFkuQY;zDFx?h(l=llztvQ6;zkqc6}Ljdb2VPVMSe(jUMHP{WzIscTuaKf3=%Z z*?y<)2b=G1@?|vNq)yp5)LBjqi69xg{j-l3aJPl|dCbQV#GC!j(x>&}=w0#KoRAx@ zjej6_$MCSgqeTgi;Jm+)3#FJQ63l&b6dWkz(bGeZ+YhiN13xxPH-M(GiTyv>rw1D2 zLy{a6D0UAL!VE{A)PLBbNICVC+_#6F7Vp1F*(G*nTs#JJkv6Z9ukApat-_Ma4avo| zQk06t7B=dyKuM9h0uPVFq$aKJ%5{;>S7+!kcV#u7@##oCH0kbk8z7m~fIoNbf6v!| zYL7uB`AbvFty0%2k5Tvd*@D-joX*-^WD<>*1Ljt$toQL9Zfy)!)za4qub;97jrOGU z*x1h|~CcPP3M*POenp=vdCdLbZWu;X4P#O5NIEVfFk+pkh?@mS>k0l+e}%dNEt= z5KbWLHj>SrQNn)L3!I?9`z^d*lL&?&Jiy|-*TED8ITvYS9-re~$@*iw;uEb1GC!KF z9sODObKwR7yf0mXgfjk67-_Nu772T@eu_F37(29aIIRNnt~=Z5Ba82k4PE;Kil9xk zWR?WoPiDYHjwh>D^YlA>%F*=899(3l=zxM#zX6J5amsmJuJ2MZfgX_6~hR z7Sdj*TcPYb*I=1}bt&fTv-kC(gSyr~<_C~sggpT>D+T%XXQVzy zQfTt|lCgb%SK`&)4KMR$YRpp17R_CiPL(pz7&(bpXPsMZMqy3z8vdw~Z1~zPq%Sxm zShWiu?J}S@toWA1{WXQyIQzt-1x^0;|IPf5sF8ojxhiG>h|2&O(f>hQRGc5ttX0Bc z`>RA(_?fcD+?aw=LT5|k-TA|3S)=Bj_R%H`mj(=@-w^oV_4z&Z9Xk7|`>?}P$ay(N0L2LxYLi%w5o;4(4}zV2#A?601x zo+*F8)?wFeo@lyZ?i;V%p~MD}(+j$FTO>r&@|zb|Q#n72<;P~6=0xKD+M4mO{GiN{ zZc?{1-^}>oa0y0%9r9D3TJnojbvNxbaHaqVDm`7YEBd$>9%-qqwGud=`n8I{ha=|w z&QfUFTz^#Kj{D+|-p^D(%p5QC;^Qqa3&o->mv@Ej7^0}e`c|+1{SqGtN883YrG5!4 zTIIx)RuIIGo+T|m8% z;FK=lhAj3fq;=z*@NIVSK(<&bk^Rj)Mv4e-ZN_zAm8}ocRU0YQUmeQxBIL#R! zR9sm*OPtkEXtq1cIcPm?{8LDT^Hvb8M}%;;?16PK(&Tz380!>>>4`~6e{bK%&}ih6 z2XB+Eqko3`$^6y}9@$(UZ9&emcia&VO0%-TRc0-o7SZEa6pKn7I zw5zI)vKHHCjCjfSyDr2nuntJ2C`KG(y2%XQo&1Ug2M3ZD?JL*crko%3< zyhxxrTP%bl%kKl}(WC5AK{@?V1`8KYUkoMxU-Bjya={0x4<_1}4GhwGdW#+?F`%?k z@sVK>4+RF0=e+)CsM=>h-15)rKBIp@@JV+U6RB9IaUC_fzx4c?p2vj6tXTI5HwSRh z;l;}kjsI~p!-W0-xA3S$K34E)Uf|9q!Iu&>ZB_=aBKxnxqe5`3f&#)LzshZ93%m=K zD7WMnJ`T3=S8N4be)`Z_ZR361}C-~2fCr5Vd-!q zn(o^*W^(vu4(5ED^M)3zwvM<%pQnDV0~ngLe9pvg*0U#zt;hK;=B=Tx^Vy6(o7!GM zhnVT+@CgXwlHqg%SM?H=2oVgBq#!nxB+B9<(pCU-84X%KF#N;z_G+wHu$@0%q=LPv zR`eP1|ErD$@T4zBrOv#pfnC>IWQ6~|T5C@InYzO-mg@YNNzqnocHvsK7_xSc&)E2! zV0`ju|5RfdW>u>T$l5wcDqYD73x@~{l%1!1U>uCw&Mswar>vIPJ%GKyVpZ|h1-3smP%Lo zW0O*gERMbghCF)M!!1{h*B$63p|S^8z{_l9(TuWZm-Y6MkAslf_C3QUNiqpGJ2INQ%@E38Y~X;` zD92ttH%FV%wPPEuz8vf;-HDrrxUhuA*@&TPp##-wlZgzN$)*;kQ*xCveuF~=+@W0z z@Q=q+wMjg|pRRY-V@6vaqY;{wb|Y(r;4iPca4>vX9*GXHWtT7A)PPx||MC?Mm9T%A z%~&erps(%ZH_g^_6ztiNaHZwy4(rqDSa(9y4E;@YH8}rrQjk|g{5VEH*!jc|sQr;M zpdQp+F z3m_iOX-&BF?Zv38OZZ><@e_M9$gw~LRn|ImL17Al{4E>c=#9!dRkMgVzc!=-Tz<9< z+JD-hxz<wG#_s-_;p@q%(x_YgZyTu$^&M+=vtP3Zw#{Oe|N-M z4UKYvzwXkSglE2-FhhW67u(KyytLCvl&dn0?nzW3*<^&*uTxyg%e_jKWp|- z<(CZzuWUo*600t(`PA=ME~JMDtUl9Fcxiat=!bL1s#VLLK~0{m%%I#yvcfHm>du$H z{U|u&+GHxW5Q!QGRH8Ra8P;5<1F+jS>l@SAt7FLy(?_q{4XTo*{M7I?cGSP43LfjA z#<}hduPSY-ntTds3NIdyjTk~oUo}WZ?_@WJ)XA&QCyGJD=7q|iLt9srAxO5^*eJ5k zY?f~-r>lM#6P4spB03v|*7oT`6@+wfer@6Jcf_GXqIPRGhPbF|!O3z#dYeuvE5TcT@YyBLR@H(T=?TcNe;Cr~n#jR&q4>g0EWr35vtsVlhGeZ* zB%CqY!{`XPxp%Jr-g5N(fDJfon^|L!lEGoJx<38Lp{mo6D{(|{2a>(w@}Qg)N=lK> z&6*H+gw()mIj*zmvTe6;A~F)Fy18GC7Y5;fnOlj&7_J(J30ICHD5c1RNL|_Sq4wG} z0)d8XgoQFErpu%hxBqYg6H0RIwwJ?Xc}A&Y=6lVc4nju|FgD(ba!oBWoj92FaJ&zx z?hYKdjKz$(z6=krp%h3R_7vu)7*c;$6q~^X|K46htHB5#=)(gorC;pxP<`^++(+8{ zkw(J8yfw}`ffz*ZdK|wKbI+~Bd$37&Fj_X#uhCcQwtcFZ@7;8#LR$s$VDK-MFW{%6 z0R8U@SU`^>{q}e^{v(`_1Gh3`Cm|)la|i&qI%R`S?!%xULj^=2Rm-3M2NqmmBRvr> z{p1>z9uZT!XPIc#bs6*#%uDY@g`UhP^*F0`BY%X|n&mS1jd*}nSM8jerr%8nl zw_S6pgZby)gJ-4_>ljD&2r(C=>c#r9iPrke&wofZWezqXXte;uD~AWyWx=g$RAYqcCgvelBm`~kDT)^rTYNxeAQDFe9c+b`?NPE%0IVgUhC zq=V0J4zaX2y+irX5@=Dk;C-1dmaI1Wa%XUcO=EIT(;r@WXb^iQ0SYQWS%uw?jp&A) z13(na@?Z6va@yI~toF!FdmPm@N~uWeA1DR;q5)(@@=voXx~*XXb?2{!-`!J}2+KOXAI4FtsG@3g8@ ze0jSROJh=Ko$XIYYawlRiSBt(n}OE?tD6OeUKy$A9!+nm{62`h17K1->!oFGSo)+= z7)v{l`@5~}GY-(?+}*x;p34QWNpFJV(;D~Qtiwk}nDcJDjpy2ll_D#!{8|(^7OfB0 zfqhk1SkMYc;9llI@Loh(UP{2OH0}VH8ckI7Tbd>X4mMYJqP~k{cS?%E!~zwGxtNjk zT_rXb?xt5HA{9_n@P_&WQwQ}0fdmKN?Nl+;v5*NBI>Dp{z9+7U6~HOuWG{6gK5y-YepqtxdT`Jq z5@tFHP2`D1ie?|Lrr?k_NyJ0WbK)&+7gt1x6|;^9(;d$RR(p?)BcXdLB82y-JAJ0# z45xqR(g)-~Wf$*=hcY3hC6>||oQBuFJ&MpnvVw~TtWV$7@&*+IAty01@3ltsa$+GY zPPB9M++S)1_YQ$KWbsd?BfqmM6aZOZgQ868nXdWfBAZC9p6Xk(F$)jMHpybBO9geK zpgcF>L)}HbcJJ^*j{RsFKK5DaPX+`aL9Qw-H*$zmAnUTXbvoUT$e@R6sb3MwfTn3{ zEOItpmkZVh7)DgUNZ-er`rwSx3=MAHUPL)DrMjo)bF*_`rb10!h0m4>?``s1q22n9d8t2@%C%2 z)1}vu_}wre5~}f`qM^B{Q56Hm+=ve0NAUNw>)O@8RIX}N<@`U?=&Ks;kPltLHEM;Wgr%eiRFo!^1U64d8A4PoG z23&oGd9yBSEKdn!}A#0L1db zZ13rtZVHhe)+G79(ds>B-ySrFsDn!rZ-59?*CJcVy+!{)spbIxE`7jXDPyOaB zUUk*coh@vG^1&+Nskt}GwVVrMs7t^tWpLl;Yfrpfbalu%37ygNO**9Fu`-( zU&(~k-Vcp`oLGBVtgDKV(mHKK5HAS*Bg6%+kbc<`p7B$07a(V(Pk7sVFkLaO@%oR2 zBZ3uz^ayQbeObAbx;kq!ccee0EP7RzSp8{ffUrRpP-2jNJwRb|tGjw3((m|e&;%ZV;SU;}u9=`L=ynuv}#f{K*%j=la zY8Yj>t5mCxnV%dPVg6K^Rrvphj>q*ai6kS5vUR*0dNcLPMn|5?iWqi0r;JjMuqEn3 zE=L4tJ(8;40{)SrzVtYX;W*tY>+2nyqgrb$>o8WC4sO2Z@cGeh+Dz%V(~)=60O9Sh zo4;@|ggw&42bUo)y!}&>@mra}`X5Rb`}0-poz{ZZE;dpF8r?Tj{dzrn{Pofn%)*_0 zKoop^`hnx-8PTKq$Mfq&xJ;ZjpSkbK+9-0Y!#u{?kTUJKSJ}|Q(3JK=V~U&esg4g1 zV|+`wmJKdFnKy$E@Ap$k5aiI4dNS~>@8uzl;0x380g?qrmA+cYolomaKq54~Err(C zo5@OYvZIp1;hqo34OGk&K3GBm11?qRF$g)(;W2li551X+JLiSBQVgu?G`r_o9D{MR&5fjdC|b&qqS2 zxlydvWkW=chlVm^zFqPpl+lHmzJXfsIX1t?bsCLAyb9Q(wT^17rON0lga50=?Zoxl z`cuHa_GRZtrbO|Dmfl)RdYSkQddd_XnA+>uu^TX|tISxpotH<8D8N+}I6s)SVs-bZ zc3CS-Ib`p>PT1XOO|PIn~dkBS`Qc~HLB>NJuXzH$2t&WH`K=B?eAdG@c|FE%dLufOcl zmK3%CUIeCKIv}o831EmP=tkJ-F!4vU=;vQOtbE4m(EHI^Fp>~X@Ej~{_8SXe9dOI^ zW_uJ*^k?w9OSL>0*Xg!^XD+WKFOJx}qur;!lFiuB$X32Pe!HTyx%_~Zcr}3^qG~aZ z*(p9dExH!K>j}U(l3M1UQG26ElQM%9*{`aY&6U!z%1+1x7|xq6>pGl|Fdh&{v>b6jU^hCtTh#R&(3Z{_la+?1j zjTJN-Y$(ZZo<4BOzhxm9HNZWoWPW43X!6y}xeBAaSScN8*Xn0_sIUU(2E_xQ`RfN% zJKMOc!nQ|XBZ0fxHDPTYx0F4o#M~Pz0vsTl7bpZC&Cb8p4`0E6g6@3=_uYrPu%F4? z#+7?ZPHp9+f?#zm9^ljnYGD1h*I#;PPcZD8Z5yt%rM3=0(^yU2@jK2a1==@Mvu6$ zEV(&|qMRO*&98^cm4j0_1~2gwyRI+3wxS#xX6gfyp7tcrjq5%a7#roiaKn3%^Gl+^eY1&ymAE!#B=t%>EbCkor8w25P-lh~8wGHd`<{X*?h}Ai4nJ z5ug33)iCfn`|_yP0N7Z&4tj|M`qyIE&y;9xXIcU5ra`UY`7UmN+9u#_Mfvg{`Uq?~ zS!-79H`=UetrYT`Y|pO|h*|fvtA4 z=a*ug7P<3>T7yl6RBHayb?)=z!(N%@9+y0w^tWOc#lq$(Hw+3;9i3 zqk8PZNtc7$25*jUyU&?n8a@NT*tawFLCQRw`-A5+r+}F7%}vk#_)%!k%QAO8?M6=) zIBzWdIx{3t^_7$DZ4qBjHc~5=ZNTRo%#mH=f=rkn9Yo=|qT$ys$<3$(t{?N6-Zvve zj4d>oI01QGaf|5Jc)NM4bYWJ+0>NUZwobZJ+R(WSq?^L~sF37a9`XRXi2U{q^f4L-T zxlod@&ZZ%vg+9&_$1VAiK{Kc6XzTbBK*r&UjHbP?f+z7<`=N-bbuoxD3S!M;O5z+K zY&mbMPIKYJnm5?v0t5-H3A?Xic?4(s0Q(WssYE?Ts10qv3I#N~sao_s7eGGhUY_qS zZoS7dQuVC_A}$$T=7)g3ULhfV=~b8?n&BF}_MRkpCU^PMUc1&aeVsLa?i(fGV$$(K zBA4Px8IcOrt3qvk9akHME~D?Ouf&8H!F{!+`x@Dp5PTQ2DT8gChPsu? zs~4@;5h~D?X8pt~OAPcyrTX<^1xFbP6LdZ;yPXAH(!dm8%*d%!}z)U~bCy|Hf9qGqA@TjQpY<`B>vcs7d z=fX6}MI5gj969cvbTxeA)IuW@uQKaw$FiPsw7^D4&O1ghB2HPU8m3848;YqUDKVF! zEGu~C#h5-~vdA4;-%5C45P|RxT0emD0R*Xg06&4Q5kyHa zVh|4~QV1I$zf9M&{@`@%YcW;e{Q{lq`o1ttt^e5gtV#?ZyBCyNUrh#WkJVA!Z{lk6 zULTgy_Op=G5YAn%?xN!!ahZ)!IKMudt7yGHV7gl3CGTfCY+;^vcr^9q_Nok(b)4xi z?a=dPWMNbTe_C5k6h$he@V$B|CTfq? zq5Pw|2yGN_TY1y-jVS2;F5qw@oYFw%0c(lSG}Or6^NPtS%XwdwgvZ9PX~?d>=~>SL z5Nlebn-kkEXtXL(32_O)tnxZ_#ThB1%Y=w6UazwhX*4o6ciaLTZfBjcm_6rv`j%pn z>P`xonLrV#HRasA!FY`m+yRq#RC9{lU!-?~+dA;tVLpqSvMh9#$$ZwV(jU$4Aja=_ z6hbQCsSuklL+jzXmrydj!3q%4xw4s3m#2pJwwlU-?55bDULUZMqjg8wius^|e92Un zpxLd1mkR`hVqQ-6f5@|nXd8-yGCE#MS)IXqI-jy(h4_j@oul{CN>FD4v0ME~biL>B zNyqE6a3HghQugSEqij<-7#kq;vmuaupJQGz!nKm~#@c-zKhCSm0iV_vF-x54@wrmX z^(0Nu$o>t(KM(_#N_J~x04H!7tI}WM0|vOgN$UA9^O=4%T3&GF0)5EK@}U&AV5Cjf z2VjNLkRA1q;AWS)~!5B5?%uc_i2xH(#mp8`MGO-1&9kcP50C(OPS3p zPP!vbe8GG;W>*YQqmR}nRCkjLuaOd%6%s5QD610f)aZ&*(v*H}Fr zR)(!RuqF2L9aQx`Ls()XRD|7nbS(kv0lvYmr?)PE0NmFLt2cqSbb;pVF6N{#P!s|yD1GO`lqh2sE%NZaJkdTmkeH=Oo zkqzO)2r!XdDQ%kwyE`Im2jhELg_T+Cd9N-i)P_on@yu#Vnl_aUD7d`S13f$PL@I7w z@yU4)B`G-co50lyq-*xx-QfhP7g<@aQ{RI*{eGfgL>0PO?!^}r_^}YXyj@)3$oBC7 zKNt~l*%w>Sl@B-Z!~(Rv+YX%Arc2b^0pgc0+L+gEXmBqv*I6I(y?Ij?9FV;!ZthfM zqbXwSgpfq#p;zo@=1n)%WlH_(O)n+|PnJr(Vm}+QmQ~j|3I3r}_KKx?g-pHd!BHo; z?j#_n@n+vRo=TfbuBVu2NU60sZBpxq1`Yv!h325g{Kpa;_Lr%ES0}lbZ@`I*pM`!1=>4h?QK24vKfghbiQI={(l6la|uT*w= zro0M$jFL?+xxnhz)~{O^=;sQf!%<c2!T5S`WF>vmQKOei^hw^gp>fVjf zfqT#jaL&cbkmGZ@E?z-#yEs+1R6|Oz`v+MMAoy(yVieo9rp`=Py1Qz|2R_%@TZtDh zDdy$mRC|k3vrFqn#4($Y_35z-DG)6l-QS=u1g}VYSWkfzt0rl@QHQP5SI@M+SSQf2 z9uhALKFJA=eX4tbFF^<)B*c|f+ZWu8@RbJsOAT#Vlv-)V7ao0=Pl;^43P6_2g1W2BkjS$Fo*BT z3ry;Yq0bL09kZ*vJH; zls+A2kM#qQwF*xkg6-+%S)DJ#l}+0d29t)dwy^QS2JaVc0tBXaIP<$p4|(?V_3;_0 z>f&}Hgn{5Zqq%Qh#qasO%FS=cFv43;4(6-XE*qc%`JG1nQs*8y6yLikEc!ZC`bEEV zuga1e^JW^wm~V)wXG^9181O+T0|)@a8wzuBIR4!6ZuWU-vC`1#YG63$?cmBLo$H$zW3l1qMayznuaAH-coQs`@B#_@S^Oo^ zT6jN+e`s07(R2U4BNm;SkfWdeDnmG+@9l9Y^N|H@BPY3|Y5E#0It;iCB~6xx^G#~` zipjRuZvduUvE}AM!m!gG5R0j4fv2DXkBaN7Ju$_WUXA(pv<+XzTRu9TWeOClRt1?A+@4+t zIXOz~2*b6f48Y9db44YqwdEfz-jpn2?=6-`YbHtHd&JC$FWwjVXHvwXBQuKj_8 z%3LYqxu5jRThla99D~Nx;d1 z6zjcR(qQ1>>l?r|<451qRbO&6&W+Jnrq3#usOoe~bja)V$m{s49Z#*0)1EOUoYA8M zeYDM`1Bh38`(kh+pk`w(K%7rxvk9u+>=QaPD0l7a9wqNAyd|8m%SFAmC|A9VFI4HU zuhex02nj{-xAf$fMIs*-R0`i*0Zg=3kk@&-rFqMfc1}k7vZEJd}eJGup0tfC#j%ewIvQ?WTDD0+7zIqI|RrE zUa=7{w0I^2yID;u+p?xQXplL zXnw}3=Uy-PhQY}2Prmi%k_%S-CVQ@tjm)EC5#La7_z%DUAw;;!_!zm>`GDPquUcQ< z6K)UWwg`XKF?+RV>!#ZSE!iwKH`EJO z^vdL`I?+upVI7r_r;Q;XxUMwpgo!&$5^v?E!z`4!bUM!D^ty7^23S;2&Se~=b|sp_ zr-s)WcPU}5oCtCkzX;0w!RYs@iu!;DSZvsLMwuW5BGeM=yh_^9tKH51N9xtPb>)uc zX@KY^Amz{xI(!K7vh3I{(rW(bz#H^22MDR{(aX_CdjiiL^~H*Iv}sm$+)RdgwDlnk z(a`1y>{X=I=%?lVXnF%KfWCX=#}>ZrVbbi>k|4J!bZ%%Y*Xgnv>5Mkb_v^My9t0Lp z4h5$?wMABVWWe+c4q@QMK(Yy}U-+ThE}6wC5q+AVqJ$HszbI?=8*pd#4=~z&sxDG+ zFKMD*c5!d60z~&xVCVECHVxp`>6h(ke4988zlm<0dQhVHLgC@j?g22XlpA$>m#k;= z#97jv^U69|+Hs`WUqbV7P_hmR#k*|U41W9~2+ zx}%j3@T%j-dFUqq5vQPgAfu*Yb#Xg!qFz>@KAUZv% zx&v!Uygj4a9L=QK3%J*8R&{xT;$YmmJJ#4db_Di*bNP~}<6Y8cXwy0BJC7fQ=?V7O$ZPi7W!+@JKV~E64Hz`K$tFTKz;(kdd(&lbGL8pu(*5Rz9+q zVFwlaUa=BMgBxz3%6nzl&hzFu*$dvxgvtYmiBW%D({?3E52d?~5p%t6GCigWw#`7z zT3%|zpr)e8WYj+}E|I@tyAv4s2Ui+9934Rt_y1-gQF#7hA(bqY(m^cuLn#yHx8rOE z4SNbyv-joY>?>l^c94SGEopr+kDp1hmFxWhD27ZC1jIGgV*}t+p_y;f6?c;G@TXr2 z2H}P9kQiP{(X;ItDedxN<3EIs==R`_LLR|vOM&7^|p+a z+$PKzKn5%1K{g!*9XCky>o;r1(&3rxEm%F>cLeyt8Ak;;54TrrMCUW?fOAzWrTja$ zZN>1zm;ztp&J8h27S1%^0TMt8RpmqsTqYyfE}ei_@o^EpW})MbRU~*ti{N$=xXpsbO0d+9UGm2dc&?LbpVl~gA zc_Zx-0RK~WRz+H2izxHGI`h$4Sk z7xZeAAWJsUwPhB4&q}<^F&|!O4Bt@T<@5&f+27^AE>L~i>YV5cP#t;NrHmYnJ%`tpIhdD=X(PUeGj7bc0Bt!8M)@{a5adH zXZ0##!t`6m-`LFK+5KPx2lpb(xk}Q-I*-N*gu_*Y#SzBvE3 zjQ~T!mKHH#gt^BInLrKz z-obP9%H!@EoX<`)UGx4+-C2m2R9U!^%kL>L3-A!9uH%yhvLKW78?SurrHKu2Uj5G= z@U08+NtDG3NkxR~9yYct`jf!==Z`vNtTAIIm|CBF1nxeQ}qyn0KWTs_-3p zzS>-W1vdQtE#Dh>XqEyh`!Y-mJ-%++jILJAE9rzXM$C_5GM)cCME?(wv1bbamncMqkx*IqmKr(b*6^eWRF&-U&u zPMLCkkHKG~*F^>MAs+aa5RN|SrO@*Xgmoq*Cd`g!h1cYBBkHg5Vd4GibTMQmES~>v z{T~z_u5V+M&t@Uetcs8}SlY}UOZbkG=c$aCwrISOd4C|3e^zn?MUnLiXs9orUt2}h z4UN|c=qsz@qbu=sU&%fgTX;6+fYSt+U|PZk70yfb7M(9aTRD|GZDIeV#h7lGW3i9- zV}C!NK|CsMj-t_w3C+q=8i>N`yy7A-uj}wA#_2|xq(}Ff6AFo?|HqW66NKab!&lW7 zj%V*RO}8Ux^<2_Mn`CQJ>fJYv1EF5n0uhRUo!BjGeXW1_bp7gTiUvgb%wL^Y;oFuE zJD6)x)4(A&aDD7!-c1&lzU`&EwQd-6eZxh6{51f@)>%sSoncgU2q zSqx?W9K;J!XqG=#)`c(($jg3j2*Qw<<510Hk}d?OpiVzN-63NK2FDLAif}1R3uKOy z((3+^1Gs-#BMh9Ba)<)ofrey& zuciw|&bFv|SGkkbGJvVJ7jN))Ka(lJRZsr8m}`UE%=$7VtES@47*2ZmvPm4hkd#+? z`BwEJ><#EKbyq~gefmX{3BBaj-_nv?up}xXB=*v0aQIMkRC5=mxmthX8uMVyXIJ9b#a*QA`r&cC;n+mh4uUU-R+1N z*-TK`H>Md9xd2lYs%_Z-Lf?X+zjrK5FSvNMKe}K!Tf&D91E}3>U1=b*7DlX2{=O*z z>?Sbcl_SFU?*3ZUZ^G^hONWDVCvHpJq(PaA^qT;dG2A~M8UqPNNAdR}r^rV<8u5?3 zNg{TSb2L|21Kd>f7q4&H@;8vyHS00%4@ltsc6)xdbe-;+1_x41JjICk}^jXnWmcL6>elrBvt`hFrjF05oKpWcvF0eWn0oDn^NhWJd{1(edw1!dFXD z1V`=nZn`Q2qfW{^ee&6wzfR}!lV_-I)PiHSV8Jfs7P0$9qFRFo)*%Tz3?eD-0Motlk z8owQXywsz4Od1(TOx-ezN&G>~Vw$o>B(-ZKdy^zbSCteOVIOuDp&A2p=_N9!HK%ps za}NE-H{c8Cfj}uC&{Fu z2(lx?eP$?>VV4Ua8NZMMgHgDrF%Sh1bN(m^7g`vRUiSBb{NqKVfysBWb*>hi96Px2 zLAtyB=Tx{5`vC0ZuCE$J`Ac=Gu{X-toD5b}n$ntepG*59ZPjUq=A3K^o{MpVUW%b% zp+zOKmrg{F*cVVP^vetHDCu9zn%Dim|3%NPL+rtNUZyif+_312&h#K>lGI+=hPWii zxS#?L!Van%@wkmO8Qrf3zF?mr&E=Z6l&P(7Ps-=SOReB<;-r}S+8Z$Xcf;{RqJ zr6+OHpzXYApsd7q+(g9E}JU|l3T*F;_KtGwvZ|l;K7g!YEcsZH6iHO?tYDR`pW;KUpzjP%;RD$y$y@{ z&Q>kFeLDLCrTUAog}0$_dBi!wK8A)tR>|GU$T`A5G`9gWKOYa@bOPIJF-_UNl&6)? z9MSp{X{^^C9c9Nw<~=EdF#=)Q7ac(CKQWQ~dq zzdk+w@~_7KZ&Cl-SN{hmgTR$cPQ|Gp^FgqV?@>8-<)j)G*avIu(|_O%ZF*uNc!+LD z7Q>IB6StIE+`kX?f59tt27rpE<8J(=j*=t=D^-^1e}FK)!B2pDKZskg1n(7`CzeXs zzmFWhK7Ex0>iH@ucZKU)7wnTL^RJ`Cuey92g8&Q31LAKVVyKoSMDBi{zWzdGvJmHm z5es2f?k$;EEOo_22LC$v{o|-69sm?V_OWO3(>v-cq5pW8$4mY;bry2Kwn<)^0}3?D zKNN}oC`C`=u~)FHhX>yK{EIaFb@KT|tk40FkG=304PD01iIV^MG{A8!`>|^=UO6ol z7dQOAb^HlU{6;8}DS)OVXW_YlztH@b=LK5SruW!!Zg{p#Tz{oZ`X7bdpgf{3avqkX z|Lt*shER$xXRA|y-q;j>|NDyduR?4|d~Z~J?dK9Ef8SL8qrntri1V=UVnfJ<^Z&i2 zKoXcktdM$`mBl1&`M=PczdQAhadabwmc9le44^*h|NpdBRD_~t7wDF0DfNlJ?q~mQ zjxCU8F3FD(b5%Fa!uQ|*^yjAoZ#c3hVqmhgTd9>irl|d&?Fehzr^-6^dK9Q((EKw` z!QZW_Tf!PU=@kOvAws+qo=YOY=ruSq%zUw$>QR~~d0ktLK1ttfGgqT{HpDghEh1Kx z2!!aoH_d@Gn-vs{8IGR&nkXM}wg!LhiH!~nts1kLg8^-zkWQXjfl~c`AOfqYuNuer z3+L_VCkH)I!uMeIJ8ncaAQA4CG?2ffTv0|Sz*PFfj^QUQBaVs~B}j7YIp2}_mz%pf zy}la`;j=la3f8{HKAUHbBr*GI(3id9+8;hDq!>;$C%nO8i}!i}FoMr~7E_<--F_r* zMuGEu4SJG$*0e+*wE=eO?v#3E{Pc|X)2O}Rl7(iEi>`_U+8#Ie^X_RmAbxX;)Y!CO zfyK@FX1HopC>5h=QYX7bm>E-_xK22#p&C{{1@`;Og4+_B7#Q#9!uX-;(wGZ_X}IpK zN%3NgxiEWko;yW8Uv{MU*C6OQzWkk;s~B7KWEfZCg<_JEl5imn(v{i--fKIZ+_j$y zc1tZ@j_Y3uT|LiL8)B4SY%2OUaYY9SrN%-ADbi=NlBCiwr}eE6!ndGnzPxq71ZI0z zyOjG_*t%mqV_#2tH+JU9CzXEG>osVFEXdnSX%%~^92oaq5bj-Z+av5TRU}&CB4v{b*yd{JUYa*L%v~hw^;TL_e5lR4_SiVWEnRx zE)QWqy~m=Yqm_Ja`! zKhD{)k9zs;89)p7eqTGX6M1k)hEfxeo|wSJIo( zub)XkaN4TKYQU%ou9?} zg{W4uOyB5uwL3Km^x-`r$Le^tc<{TnJAr}Yy7}8(G2+#C13~6sms~C!uUedI7uA3V zkWmL*X`^jV*aR!fynl#DvG5qVM0B+h00u@=ieF=p@a*yDR>I>fSu_}V2O-5`1bt4f z=52oyXc;UUi_)RVZe$kRN;l-qPEaBV*HI7Kh-itl9ZtE9nw~nYqeaEJh%InpwrCV%*dRR@Sr{UGLr)5#18SnzCrZni7g`~PrDwXk+^E+h|CR% z5C!JB;E-gp3A@=8+BtFZF}r!YFp?Sl{Q6b6<%tpc?1P9&!$Y=yG2Lz*X>~jM=RR|y zla8~GbnD zJ3&KX|J%3O2CpPjTVPwa{Oi(VE)3Xp?aSv`h!9&5qzzMvriTzBZESuD)(Eeak3?LA zrUvhS??~^l9}8ia`w6wW6}&WYzFvVkApJ}e>ue2s`7`P@-`Lyirqq=!%jos2eX+)A z0(wxoJtj&-8luUZxk%c9v%ekE-luI76hjZ8`BxG|`4ozirj>eEBD6KnR!0IGhueH% zN88OB?5G!{SM*WWenRs+kC{|T?$r9`8QIqDd(B;b-;>T;G401!)>-~ABAJ#uL;`sa zKaTD{#T*iSaVq16w?^%WI%JeAIPGvuuZC$W6ru)gWZn3A`N2ox(rdO|(VNsLjZMin z$-TVPS`&axZim58Y`wV3MqLP{iqSnh%LL>O-V%hpv^OoyUTu!2u%Sj zBBw+iS;9{`A(c;Th8$86PoZwEUyZez5@U{yQy-8Lp64kp%Olj9D?K5{6aMl9jkv4k zy|a+XJMNHEnjA*-%eD-in7}F>ekz#G@v- zPDoGv15wv&#I-Am;wPwc%TTKiaJ)B92KTuV=Qm8re-f94tVLbOk4bmf1J;}xtwspXIh&=a(pf@pi*)?mPZ6dU0FiFVk`iH$aLZlTMpJa3|bzkZLPs5Vvi&l0hSLLxA zH%aTV0c3SZ~X&HJ$fipB%DI>pkFHh0eN75ig3UE#}+k(dO&^q}davm@$@91I&yI9dr8|xZI_R75WHUhgQCh-bw~E&xnol=rXgAUOTA8=`+k`Pp zZD2jG_8wlCQQ--G z^t>EbaQ7H#d{kcEUS*`%189&87(&#W=eZnE=fQoaXaUEa3&r+^a(UmMV20!r?5|vS zw%i;DutGtkiPb}fC0?*pBX_orS|};H4Z$3NDAq`!SlW293T%9$#iY=i&=9Z?8pdm z$7?cZbf0lcUu)pMK7<_ILM^t&u+3Ev-3G69hhM<#on<+ZEl(GGgx6ofV86q=TZ%yW zHiby%%Ia--)maIghDmF!Px0tcDi}jthk~p9RK2Up1BUg~kXJJtd>zSPl_d$=ry0>Q z-XY=8-r_;diOP;6%0IAwF=`R@Ae6*jZAB4>UP6rAp31=zA~0RIj0kTlBpW{&>bEFh zK0+B73BD=zKiamaf?XDJX*Zj4r)^0 zV6QFC4nC>mMa>8(=t^kz+-PUKOKYlOTTDBi+cGM$K&{m}6l8P8G2>?#-lwc5+@qS+ z-B$E*qgrC_Gc6#!cK#G6JL>8w{UZzs-Le|-8A0-8eS9*jc~6+y(Mze!4c%1qX)J0X z6uuySO>XJyn;b^A6JR6UDAn#c8+1etW-t>PLM12SYmg*9px$>l!fPU(Gn0>p3b#NMikcl82 z;3PDRs)L`)#RLL1p`7H5hu%dxy+Ry+fqeZc;N}}vH8-L>`aHjzS&;Da&4wBoyPJz8 zZ*Ld}LGqy;&y2~os^nBjOZHmfffqS?m!1WFPW;LBivgxWq@C*T*RwJBj@L17z=$W= zH_euOhD{kE=Fqp??yqU+w{`qOAB-s++EoXY1qELa54E%kOiSH%m=?{4`#T2b zm=Q_ts0}C0O}2H_gu`bh*%i^!k!b!h!s`MScRgY=eC`Ay@5tFvF@k<1&qqk>fwZWj&?vd`^3af_UhUpv5%C+aXA3Nh6`7%a=A1 zO$i$q>7V5Q{lIA5z~&1p!_OnSN%ZEai1gD;K}bH@;dr`F4yxp84{6w@ler1M{9A425kV6oA7; zwr#F5nu};cCP-0N3hzaItANmsMZP{Lxzg&XMO6$M;*3x47!m`~UoBtA;_O^@vQmj+ z5zKxWO-`CIUhiGDJ_d^2cm}u&$uts- z-=N%W3J@}4mqIt<(fccBZrY+yovh0e966eU;7qq+hWIl4SIi#P{XUdc+dB+GTrq#H zYfVQ84Yx$&p>)$N*Z?nce&33|r>|MhZj2&MLuhZn2JxTa@b&xKJMWh#(EFmotB2S9 zvsV?{ZBmSTDActQq3oRZtcLywrGe6?v`)T00Dh8ONlf!UDca)9kcrZ;i^(d zS}KCjVPK>^kmXk2Qj|S20_0ULL9EMeE0tl8!LK0p!G4z6z7Jmga)amzhmQ5S@7?lHDE%{uoE6d;+IvnQOH9?9AWHFgftH zwbyeLhjWjYbTH<dyTW-KqA`dX3xfXr_jO=O3F6vru>2!&kf_ zoys&DDKn8m4!MN=Grc0@u+GWfMf%!jlI>4C@k){FcoXs(XTRs(uHyp@AV?lx! zK|jh|h*$jn%8oES?+vN4o~lJW;E@cmB-c0woC6LcmvdbA! zXQ4^16@t(lpO6Zh5X$HT+S*TIWR3`OVTf(@uJH1j?pIQmW6x-WgaRXmm$_N7(N6MW z#R7$J+Tr&E7!jT$4lO3$yu0$bYwcJEsLP=kHyF9!u|~7t&+Cz$g**u0KT&r?<*;B= zpk=^3zv>(ox?;61!ZzI>eft zhcoL#nl~Ifkz7rvT6ce*thDa zDkff$mOi6fuo=hfCYbh^=b*jnOtBX^?7o>@{vjx(Agwxfy>hTuU7V;*claOy>}dgjhQ2=(U4_VNdzcvfp8>o~WT2IV~GD(*m!& zB5B#~EJWq?6dU~^ z&kBRGuw*!HqZQNld30=xKzc&GqumP1L!@}EcA9P<}|F@<;}A>8GrvdFmoeIY)TXYBJ{!O4>+=Q#ID3*#w0 zn0&BAUD<~yXbIYbi(l@9?*W}cB$YIL9p+UmWlTg!q<0O%DNlZ6L%-+v z3N-eLA^+dO&;DHn*q9@~T+D-Z9CKGz|sR-xaqy{tKzuKgC;KtX2CW0pJMRFm~79;OS$07MG1Ad=5i186_lxV=JBzXQspUefi zPu;-BdYF4)IFG|mfr1JuuXaOR_GzE_8T|YZFR#Tsu%vRfGXkkxK8N_7QI4kY+dph8 zujdH(xGO(`OlaRyh!e2vs2khDJR87A0dz0yM4)pHUx4-<1%DNcv&l>khD!1ccn0x# z3a|O_^qi?bFygqlgfYxhH~z*FPvq?LGhTgoQ3j@U1apHq^@Mt{tn4>D?PBKD_U+rp zN8ez$wFY2;s3hPVNVgdar~`$6o&=+yO=T9tZXj)*u6U0gr0+Fvpf1l~Of-S8H3xh> z(eEiNCPo``eBVwU%+8nq^RDTS$ls^GVeVRuh_%r!=9vnc0waZEzikIz`^PhGK=c{k zz?3=MSReH6t20;J`B(NLTE?!m54xmqi$64^F+%_C!t{6`wo2tH7s@$dB_BUrnG=gU{Sepz;2zlPD+s zOwMM6b1s!M{jr`rCHO;~c`}ZhI(&ZfS&^35qg;H>lTJlJ4&ct$Dkh-6aDBG|&v7c} zGNY`-87W14kS|XZHbS}_Cv%{n;HjgCK>bz(?hpd1K)6G|YO>x4V1nhuXoLbxatB~| z9M**g1vTnHyTZTBC^tlkBFjlf-$Bq=BO-5Xh zAiOJtwSJgPJ7Uq01e0t!xG7wbhj#(+C^p?r!`()v#a-~{@DLn=vQU8Ht}>E!QK%Ix z;JMSY9uKr(C@&W<=fKAWXBpTl5s$(~GEVYbtRUiZpnm|H^)Tf|eJF4kH|Sh&(9PW_ zfzb6S%KRDV++9Bm+Y=9pkAG3Ow`s7*+m8xR1J4K8An@^_MB#5FgdZwDN(Ti68LW{% z1unW>h?5_@6wbJ?*oQ^^75FKjFJrohvk|6RqC|=KkUDQ$kg%K-{*Z*{!=1jRs3+gG zjSrl6pl*COkMHRDqzPpXj|Td~&LqgZkVbWsmu<9xZW`3pbNm+%)1}yD;I4-YI}cnm z%m5D;0VWMBD!8*M;Ng3TT=erOJC6f zbYVZOhCaq!0n_%T0FE!@MeMm4Ga>MQ4aX3axi$^_ywHroD;F5-+w9|1AaGYDG1{D` z7D}N_79syi;O9l4p;(|AgR#>DW&Rv|9|E_=NSpU6!1DSeLfn(!JBf6Pz%)QQ+?nJn z7WL2vxXW)MfPMhHIk1okLfv?Jf&wg+4L*Gu`LS*WBgEsY6+AsbMF1CqTOmYoQFaye zT9REP}oSUn#hG@5OuIPlNIW zY6sR~{Har?$VVJFru>a-mfPfu2*p??U_;s}q)q-L9KXoBIKtOJnHCQ?*W&yFzt0C{ zV%--w1N^0hym+d~2TR1IB73bDY(NoVQ5xH=iM?h!4uf6F7z!6wkZ_I$x>E#=eH~y@hiB1%YoL=#h>?q+$PX zoI2+q*3FD#q-B+jN9BF$hCJDqOdkd>>%0jx_EoPw0F1cP=rcdFAmHa|E561=uQ8|x z1y?>Fs3aJMK3yG#W7>p6SxN<|zqzm?`l3~KQ#V8iSay`z;0l{%`n0I=lodMfFELD{^792A0=m{MpoJFyrGR0OZVGhilTh55kWyz4@cdA`E-X*1N@oKnFv zDhtM4a$v$an9r3!ZHHt$AJ4c!`N{9~yhhr5WiT1r#XWCvU9f^}h7?vrRbKKTUDIBN z4jtk;!e`zF566esJb*g*jN$n4<0nL(k**yupZ_1)i{pbQ8@tktzROn%5LLLgJr%dZ``=yGfyEcuD^J{1n<=1nyVwy-k_yUU zp5ml(2rT+h1Y#cnRUpJZrkbK31TdjdKr)L1JdC*b<Lm}E*coV8)4i@tq6W=u|5Y2f>MZUJ|jkqxP-~mqm{*C`P}#JchT@r*s3)+v;qC1d26=4y8shcC zq<#SQS}d}8e49t;zeO3gqs$Zn%>x?o`5>{}d>AZCmo6P2fU_763&11bUyibJXQ&Jw zBo;5C-u!@v4=^rj2crCDR~+U3l*$)yB^oqn5QRT3@U4d;cY0XPuTe${jhTT9-%sVP z^mC{uPd)Ix&3DR7nKG?M*?iIrgq=Gr zkA-|FfSL!bhv;^o+~32=3&to2k&ns*kRm?7iVs;n%y)spUF0$76WsCi;0TO-`d%vh zfj>SLqPw|}fZGVOuua_&hgeXNkj#zK0+i(`^f8_!Af|kG671d|;6TCDEbUWf=cyq`e1q z=TS`dTSW2tTtc2_Abeg%{npcDv9BP=n( zdUAe*+vkHYw&8B{bDrp8TlkDU)R}XRQDk9$$cscSA>NM=;HV%?j@?n-9l<`%OSpLQ zfy&)P6w<(F=HyPDI&uDvkNJ#>LeqvwhaaYG%~57PBhfO5&oQtJ`SN6yXI=uGH=TdL zz6RcQC?nt5rw|a;I0U=r<247Nye2*bc#eNES>^FtH{@9gA^y&HB z@|mBJA9u7<0DJ41#60bWidSn~c~k7lQA&aSa}s@? z$Kzke{Li-KFTE_m_~%@h5_R+Hr>JvZr1Ku*!A+&L749>Z5FTi}`tTwjGY24TDgsQK zu;0MX^1Q|Tfcp*LHUKpHhxhsPNB9lQAAI)jgyM{M*$CQ{JNHy5Y(n@cgmX@0pErt2 zdm?aC8N!POc?lpDLcDjVihF*!?L3e-Pvkh~UC;9s`TO)Y%yE20;7ee`S2;MJr3VI! zkXE3!Loy=IKKrcKxIsJkjPDBGh#9lP3yz+2#$jH;w5B78m*x`#0yi|6y+A z^H5M8Bfz5$?8lz0Q=L$288M8zyi*%&9dt$9F;d5{vhgus34@oi@sM81_0m+%h2F zEO2Lo;qf-Xf`*I6Dq&$^c|qg-1AK_p0&gz(ucA!UCg+1cp|q;W000NTNklw5gxcr^c<|)`CYeBcj)eqYHY_Nu#W_hHgEDfL%Y~=Q zakv+N*9H497mjdq_tn%DX|;o)mYi{uL1^g2G*SL!czFH@USiJ&TxGa3!K{NB3&VG5 z8HdM3T}+oA%OE^8aYNdCXTBsBXJP#>O{Fm*>9-oL>G zbOoea0`=hO01Dflwl@WId;uH2OV11XScX+dH-UR$U^wC50pt4m>#tAIW?RB0{w~Cw zM_%|{yzmn?cPm`5JqCFE3t^KNZ1P>>tniy-=_c`;UsQx~40$L>o2PWVcK;ZN_&+f{ z^g;G__WMrA&u^bb8thkJFdle#W4aj&puZ3Q)37%Hs}yL9((qTO+uh$d7XCrFCqCSZ z!NU_jOwMCK`w?S;!fYM5bHeOEpx<^t*as+gCgf8P=~d>dChVV}dyQ*^G2f3t=lj&= z=Lq5Mjk>=9n>%lO?SXYq2ik0upQm)juq}|c&j)dmqyPAfQP6$TA`NMJ%@0`cao6dZ zd%i0Po^%*%Tm=uYyhp`OHg3!zUe6J}jskw-W5VI)FAYRicKe*R3;cO7 z|M3#v6v&gW@v$#FkLOo5;G7=ed|sM4iy2^dlF!4GC{jMH_ zqIBq+#Sli`=V5av8^Jd}Nc$hqvtcePhq|*r=R}#;kOOqayRAHs)>fqLZ{9V}SLET- z-_Tch5{$3b@O~i*!{<+&I58d17Qb!BveNh&H$LMV%FRcRelN5oPjY>Oc+)svp`2{9 zk6}kzUf|_vFYpGMxBblnK4Tblk8pMFe$6spbf zbWDg>Ejq*QlyN@HIN>|<#RG?zH?DIFn5HN!A9;-jU-0|9s}6FMA3V_Z0TUk0i<0+N z7*MVYR=HuHL0xzffeTG8nl~d&uLT|axBNj~K4~I9AM9e90dZ#81H|L5-4ZUw$rpgf zkb$#L-!lPzzPfemMrBrfw9jUw`yc9Ia3d@02^x0`=)LQP3nP?^J4}At4nhHUJGfwt zR@s;yu<_}GsAr7(G;v{DFk`xb8+T0TjkZDEc+Y~F2O?_+CdhIlAMUo&>oLx8?Fk6r zKwY`JWJ0mClK}#cQA8O+48QDYSy4f=W63_d%s$_d4~hkzKzbJn3?D@g!y_|D-)HPO z`KdGjR)_3#V11a%D$ z;hCSicJ2BU=OmDkS8_VG7wqc540j&|M=BSx5u{Xh#ah7od)KIkXW ze#S>u#JF5Xd%4l)HlfZw&u%-9x)p)gL_ie?v5BUpr3fej_Y#4kSkU`B(t2BEI*z~4 z(9rZqB+f08P^L&fSeTB*0@Iv?iInj^2t+d+zuWBq{1(Sl0TB}@u|qlu3JU%VrdKrV zC~_ArCl>8Ig6$Qio_mh~c8`CCAou#cFUhTy0K)5)3>h-;V#rqLe<@HGb2{+WQqx#b zA%N!+l?JIW7qvPTBtR7FjKtZ;_U;n$C<#<@~!t?0@@doaE=RH=d%@YGi z-$!u}tC?tGihv@Z2-px%1%eF>bt(difFhs>C<2OrBA^H;0{0jJ&0Y~u1QdZdi@^T_ p00960IS&pS00006NklExtract sections"] - DGS2["2. Generate Q&A
With verification"] - DGS3["3. GenSelect
Self-consistency"] - DGS4["4. Evaluate Quality
Judge-based scoring"] - DGS5["5. Aggregate Results
Combine datasets"] - DGS6["6. Difficulty Estimation
Stratify by difficulty"] - DGS7["7. Prepare Training Data
Format conversion"] - DGS_OUT[("~800K Q&A pairs
Stratified by difficulty
Work in progress")] + DGS1["1. Preprocess Filings
Chunk SEC HTML"] + DGS2["2. Generate Verified Questions
Q-gen + Yes/No verify"] + DGS3["3. Generate Answers
N candidates per question"] + DGS4["4. GenSelect Answers
Best-of-N pick"] + DGS5["5. Evaluate Answers
Judge (multi-seed)"] + DGS6["6. Aggregate Answers
Consensus answerable"] + DGS7["7. Post-process
Clean + rename"] + DGS_OUT[("~800K Q&A pairs
Single final_result.jsonl
Work in progress")] DGS1 --> DGS2 --> DGS3 --> DGS4 --> DGS5 --> DGS6 --> DGS7 --> DGS_OUT end diff --git a/docs/maintainers/containers.md b/docs/maintainers/containers.md new file mode 100644 index 0000000..b91c84c --- /dev/null +++ b/docs/maintainers/containers.md @@ -0,0 +1,137 @@ +# Building & Staging the Cluster Containers (maintainers) + +> Audience: **maintainers / builders** who produce the `.sqsh` container images for a cluster. If a maintainer has already staged the `.sqsh` files on your cluster, you don't need this page β€” just set the container paths in your cluster config (see [INSTALL.md β†’ Setup Containers](../../INSTALL.md#setup-containers)) and continue. + +NVFlow uses five core containers converted to `.sqsh` format for running on Slurm clusters, plus a CPU-only `nemo-gym` worker needed only for GRPO / DG-SDG (see [Gym worker](#gym-worker-cpu-only) below). Of the five core, **four are built locally** from self-contained Dockerfiles in [`dockerfiles/`](../../dockerfiles/) (`nemo-rl`, `nemo-skills`, `vllm`, `vllm-grpo`); only `sglang` is **pulled as-is**. + +## Build host requirements + +The `docker build` step needs **internet access** to pull base layers, source from GitHub, and packages from PyPI / NGC / Docker Hub. The resulting `.sqsh` files then run fully offline on the cluster. + +- **Docker Engine** or **Docker Desktop** (any OS - Linux, macOS, Windows/WSL2) +- **`docker login nvcr.io`** - required once, so `docker build` can pull the NeMo-RL base image +- **`docker buildx`** - only needed for multi-arch / cross-arch builds (ships with Docker Desktop; on Linux: `docker buildx version`) + +> **Note:** If your destination cluster is `linux/amd64` (the common case) and your build host is amd64 Linux / Intel macOS / Windows, the default `docker build` works without `buildx`. + +## Required containers (5) + +| Container | Source | Tested Version | Action | +|-----------|--------|----------------|--------| +| `nvflow-nemo-rl` | [`dockerfiles/Dockerfile.nemo-rl`](../../dockerfiles/Dockerfile.nemo-rl) | base `nvcr.io/nvidia/nemo-rl:v0.7.0`, Gym @ `33ef60369` | **Build** (Gym venvs baked) | +| `nvflow-nemo-skills` | [`dockerfiles/Dockerfile.nemo-skills`](../../dockerfiles/Dockerfile.nemo-skills) | NeMo-Skills @ `e06c9b90` (tag `v1.1.2`) | **Build** (see Step 1) | +| `nvflow-vllm` | [`dockerfiles/Dockerfile.vllm`](../../dockerfiles/Dockerfile.vllm) | base `vllm/vllm-openai:v0.22.0` | **Build** (SDG/eval) | +| `nvflow-vllm` (`v0.20.0*` tag) | [`dockerfiles/Dockerfile.vllm`](../../dockerfiles/Dockerfile.vllm) `--build-arg VLLM_VERSION=v0.20.0` | base `vllm/vllm-openai:v0.20.0` | **Build** (GRPO rollouts/judge) | +| `sglang` | Docker Hub | `lmsysorg/sglang:v0.5.10.post1` | **Pull** (no custom Dockerfile) | + +> **Note:** The four custom worker images (`nemo-rl`, `nemo-skills`, `vllm`, `vllm-grpo`) are **built**; only `sglang` is **pulled as-is**. The custom Dockerfiles bake in their source, pre-built venvs, and `tiktoken` / `openai_harmony` caches so they run offline under `enroot`/`pyxis` with no outbound network. + +**Optional containers** (not currently used by any NVFlow recipes): + +| Container | Source | Action | +|-----------|--------|--------| +| `megatron` | NeMo-Skills Dockerfiles | Build | +| `sandbox` | NeMo-Skills Dockerfiles | Build | +| `verl` | NeMo-Skills Dockerfiles | Build | +| `trtllm` | `nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc8` | Pull from NGC | + +### Gym worker (CPU-only) + +The **Gym-only stages** β€” GRPO `prepare_data` / `prefetch_cache` and the DG-SDG gym stages β€” run in a dedicated **CPU-only** worker, **`nvflow-nemo-gym`** ([`dockerfiles/Dockerfile.nemo-gym`](../../dockerfiles/Dockerfile.nemo-gym), base `python:3.12-slim`, upstream Gym main `33ef60369`). It bakes one venv **per Gym component** (`gym env start … +dry_run`; `equivalence_llm_judge` + `finance_sec_search` + `format_verification` prebuilt, others build on demand) into `/opt/gym-venvs`, with Gym source at `/opt/Gym`. Stage it if you run **GRPO or DG-SDG** (SFT-only / eval-only runs don't need it). It is referenced by `my_cluster.yaml` `containers:` as **`nemo-gym`** and listed in [`cluster_configs/containers.yaml`](../../cluster_configs/containers.yaml). Build multi-arch (amd64 + arm64); `GYM_REF` is a pinned SHA, so layer caching is safe. + +### Launcher image (optional, airgap-only) + +Separate from the five **worker** containers above, the **`nvflow-client`** launcher image ([`dockerfiles/Dockerfile.nvflow`](../../dockerfiles/Dockerfile.nvflow), pinned Ubuntu 24.04 base with Python 3.12) bundles the `nflow` CLI + baked venv so **users in an airgapped environment who cannot `uv sync`** can drive NVFlow over an `ssh_tunnel`. It is a **launcher, not a worker**: it is *not* referenced by `my_cluster.yaml` `containers:` and is *not* required for a normal (`uv sync`) install. Build it **multi-arch (amd64 + arm64)** and match the client/cluster architecture. See [`docs/remote-launch.md`](../remote-launch.md) for usage. It is listed in [`cluster_configs/containers.yaml`](../../cluster_configs/containers.yaml) as `nvflow-client` (release-tag placeholder). + +## Step 1: Build Docker Images + +NVFlow ships self-contained Dockerfiles in [`dockerfiles/`](../../dockerfiles/) that pre-install all Python packages, pre-cache tokenizer encodings, and pre-build virtual environments. The full build commands β€” single-arch, cross-arch / multi-arch (`docker buildx` + QEMU), and the `sglang` pull β€” are in **[`dockerfiles/docker_instructions.md` Β§1](../../dockerfiles/docker_instructions.md#1-build)** (the authoritative build reference); per-image `ARG` version pins are in [`dockerfiles/README.md`](../../dockerfiles/README.md#version-pins). + +> **Tip:** Keep `NEMO_SKILLS_COMMIT` consistent between `Dockerfile.nemo-skills` and `pyproject.toml`. For optional containers (`megatron`, `sandbox`, `verl`), build them from the upstream [NeMo-Skills Dockerfiles](https://github.com/NVIDIA-NeMo/Skills/tree/e06c9b90/dockerfiles). + +### Step 1b: Sanity-Check Images Before Conversion + +Before the time-consuming `enroot import` step, run the smoke checks in [`dockerfiles/docker_instructions.md` Β§2](../../dockerfiles/docker_instructions.md#2-sanity-checks-blockers). Each check is a **hard blocker** - if it fails locally, the image will not work in production. They verify the offline-critical pieces: `uv` works offline, the trainer's 7 baked Gym component venvs are present, `tiktoken` / `openai_harmony` caches load with `--network=none`, and `tzdata` is populated. + +## Step 2: Get Images onto the Cluster + +Slurm nodes usually have no Docker, so `enroot` pulls each image from a **registry** (`docker://`, recommended) or loads it from a **saved tarball** (`dockerd://`, for sites with no registry). Tag/push and `docker save` commands for both paths are in [`dockerfiles/docker_instructions.md` Β§3](../../dockerfiles/docker_instructions.md#3-convert-to-sqsh-for-the-slurm-cluster). `sglang` can be pulled directly by `enroot` β€” no push needed unless your cluster cannot reach Docker Hub. + +## Step 3: Update Container Config + +Copy the template to a personal file that records the registry / tag references the cluster should pull from: + +```bash +cp cluster_configs/containers.yaml cluster_configs/my_containers.yaml +``` + +Edit `cluster_configs/my_containers.yaml` with your registry paths. The YAML **keys** (`nemo-skills`, `nemo-rl`, `vllm`, `vllm-grpo`, `sglang`) match what the workflow code references and must not be renamed; only the registry / tag values change: + +```yaml +containers: + nemo-rl: your-registry/nvflow-nemo-rl:v0.7.0 # built locally; Gym venvs baked + nemo-skills: your-registry/nvflow-nemo-skills:v1.1.2 + vllm: your-registry/nvflow-vllm:v0.22.0 # v0.22.0 for SDG/eval + vllm-grpo: your-registry/nvflow-vllm:v0.20.0 # same repo as vllm, v0.20.0 tag for GRPO rollouts/judge + sglang: lmsysorg/sglang:v0.5.10.post1 +``` + +> **Note:** `my_containers.yaml` is gitignored (`cluster_configs/*.yaml` pattern), so your registry paths stay local and won't be committed. + +## Step 4: Convert to .sqsh Format + +### Option A: Automated Setup (Recommended, for Option A registries) + +Use the setup script to download from your registry and convert all containers in parallel. Pass your personal config with `--config`: + +```bash +# Run from a cluster login node (sbatch requires Slurm access) +sbatch --account=YOUR_ACCOUNT scripts/setup_containers.sh --config cluster_configs/my_containers.yaml ./containers +``` + +The `--config` flag is required - the script reads image references from the specified YAML file, pulls them via `enroot`, and converts to `.sqsh` format. See [the script](../../scripts/setup_containers.sh) for additional options (`--platform`, `--force`). + +> Output filenames are derived as `-.sqsh` from the YAML key and tag (not the registry path), and any image whose file already exists is skipped β€” pass `--force` to re-download. + +**Check progress:** +```bash +tail -f outputs/logs/slurm-containers-.out +``` + +### Option B: Manual Conversion + +Convert images one at a time using `enroot` on a cluster node. From a registry, use `docker://$REGISTRY/...`; from a loaded tarball, use `dockerd://...` after `docker load`: + +```bash +CONTAINER_DIR= + +# Use -.sqsh so manual imports and setup_containers.sh agree. +enroot import --output $CONTAINER_DIR/nemo-skills-v1.1.2.sqsh \ + "docker://$REGISTRY/nvflow-nemo-skills:v1.1.2" # from a registry +# -- or -- +gunzip -c nvflow-nemo-skills-v1.1.2.tar.gz | docker load +enroot import --output $CONTAINER_DIR/nemo-skills-v1.1.2.sqsh \ + dockerd://nvflow-nemo-skills:v1.1.2 # from a tarball +``` + +Repeat for `vllm`, `vllm-grpo`, `nemo-gym`, and `nemo-rl`. `sglang` imports directly from its upstream registry (`docker://lmsysorg/sglang:v0.5.10.post1`). + +**Two things to watch for:** + +- **Registries with a path component need `#` instead of `/`.** `enroot` parses `docker:///` such that everything after the first `/` is image path, which breaks for registries where the host itself contains a path (e.g. `nvcr.io/`). Use `#` to separate host from image path: + ```bash + enroot import --output vllm-v0.22.0.sqsh \ + "docker://nvcr.io#/nvflow-vllm:v0.22.0" + ``` +- **Filename colon.** `enroot` writes the Docker tag separator (`:`) literally into the output filename. Either pass `--output` with a shell-safe name (as above) or rename after import: + ```bash + mv "nvflow-nemo-skills:v1.1.2.sqsh" nemo-skills-v1.1.2.sqsh + ``` + +If the cluster authenticates to your registry, drop credentials into `~/.config/enroot/.credentials`: + +``` +machine login password +``` + +Move the resulting `.sqsh` files to your cluster's container storage path, then record those paths in your cluster config. diff --git a/docs/recipes/finance/README.md b/docs/recipes/finance/README.md index 15d557e..2350a4a 100644 --- a/docs/recipes/finance/README.md +++ b/docs/recipes/finance/README.md @@ -8,19 +8,19 @@ End-to-end pipeline for generating synthetic financial Q&A data from SEC filings **Two Independent SDG Approaches:** - **Template-Based SDG:** Adapts seed questions to different companies/years, maps to relevant context, generates and filters answers -- **Document-Grounded SDG:** Generates questions directly from documents with built-in verification, quality evaluation, and difficulty stratification +- **Document-Grounded SDG:** Generates questions directly from documents with built-in verification and multi-seed quality evaluation, emitting a single `final_result.jsonl` **Production-Ready Pipeline:** - **Data Generation:** Uses GPT-OSS-120B, Qwen3 (14B-235B) models for synthetic Q&A creation -- **Scale:** Processes S&P 500 companies (~100GB filings) β†’ generates 1M+ Q&A pairs +- **Scale:** Processes S&P 500 companies (~100GB filings) β†’ generates 300K+ Q&A pairs - **Training:** Full SFT pipeline on 256 GPUs (32 nodes) with Qwen3-14B - **Evaluation:** Benchmark trained models on financial reasoning tasks ## What This Recipe Produces -- **Synthetic Q&A Datasets**: 1M+ high-quality financial question-answer pairs +- **Synthetic Q&A Datasets**: 300K+ high-quality financial question-answer pairs - Template-based SDG: ~300K pairs (used in production SFT) - - Document-grounded SDG: ~800K pairs (SFT integration in progress) + - Document-grounded SDG: additional pairs (experimental; SFT integration in progress) - **Fine-tuned Models**: Financial reasoning models trained via supervised fine-tuning (SFT) - **RL-trained Models**: Models further improved via GRPO reinforcement learning with LLM-as-judge rewards - **Evaluation Results**: Model performance on financial benchmarks (SFT and GRPO checkpoints) @@ -78,14 +78,14 @@ End-to-end pipeline for generating synthetic financial Q&A data from SEC filings β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ β”‚ β€’ Generate questions β”‚ β”‚ β€’ Preprocess filings β”‚ β”‚ β€’ Map to context β”‚ β”‚ β€’ Generate verified Q&A β”‚ -β”‚ β€’ Generate answers β”‚ β”‚ β€’ GenSelect answers β”‚ -β”‚ β€’ GenSelect answers β”‚ β”‚ β€’ Evaluate quality β”‚ -β”‚ β€’ Filter quality β”‚ β”‚ β€’ Aggregate results β”‚ -β”‚ β”‚ β”‚ β€’ Estimate difficulty β”‚ -β”‚ β”‚ β”‚ β€’ Prepare training data β”‚ +β”‚ β€’ Generate answers β”‚ β”‚ β€’ Generate answers β”‚ +β”‚ β€’ GenSelect answers β”‚ β”‚ β€’ GenSelect answers β”‚ +β”‚ β€’ Filter quality β”‚ β”‚ β€’ Evaluate quality β”‚ +β”‚ β”‚ β”‚ β€’ Aggregate results β”‚ +β”‚ β”‚ β”‚ β€’ Post-process β†’ final β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Output: ~300K Q&A β”‚ β”‚ Output: ~800K Q&A β”‚ -β”‚ [Used in SFT] β”‚ β”‚ Stratified by difficulty β”‚ +β”‚ Output: ~300K Q&A β”‚ β”‚ Output: (experimental) β”‚ +β”‚ [Used in SFT] β”‚ β”‚ Single final_result.jsonl β”‚ β”‚ β”‚ β”‚ [Work in progress] β”‚ β”‚ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ @@ -139,16 +139,6 @@ End-to-end pipeline for generating synthetic financial Q&A data from SEC filings ## Getting Started -### πŸŽ₯ Video Tutorials - -> πŸ“Ή **Coming Soon:** Video walkthroughs of the complete pipeline -> - [ ] Quick Start Demo -> - [ ] Download SEC Filings -> - [ ] Template-Based SDG Explained -> - [ ] Document-Grounded SDG Explained -> - [ ] Model Training & Evaluation -> - [ ] Production Deployment Guide - ### πŸš€ First Time Users **[Quick Start Guide](quick-start.md)** - Run complete demo with 7 companies @@ -172,7 +162,7 @@ Detailed technical specifications for each stage: - **[Template-Based SDG Stages](stages/template-based-sdg.md)** - 6 stages - **[Document-Grounded SDG Stages](stages/document-grounded-sdg.md)** - 7 stages - **[SFT Stages](stages/sft.md)** - 6 stages -- **[Eval Stages](stages/eval.md)** - 9 stages +- **[Eval Stages](stages/eval.md)** - 7 stages - **[GRPO Stages](stages/grpo.md)** - 10 stages ## Quick Command Reference diff --git a/docs/recipes/finance/quick-start.md b/docs/recipes/finance/quick-start.md index 0e85fb5..ffbb71d 100644 --- a/docs/recipes/finance/quick-start.md +++ b/docs/recipes/finance/quick-start.md @@ -66,7 +66,7 @@ Evaluate three baseline models on finance benchmarks to understand pre-fine-tuni uv run nflow list-stages --config nvflow/recipes/finance/workflows/eval/demo.yaml ``` -1. `prepare_data` β€” Prepare benchmark datasets (SecQUE, FinanceBench) into `nvflow/recipes/finance/datasets/` +1. `prepare_data` β€” Prepare benchmark datasets (SecQUE, FinanceBench) into `outputs/finance/eval-datasets/` 2. `qwen3-4b` β€” Evaluate Qwen3-4B on SecQUE and FinanceBench 3. `gemma-3-4b-it` β€” Evaluate Gemma 3 4B IT on SecQUE and FinanceBench 4. `gpt-oss-20b` β€” Evaluate GPT-OSS 20B on SecQUE and FinanceBench @@ -80,12 +80,12 @@ uv run nflow run prepare_data --config nvflow/recipes/finance/workflows/eval/dem Verify the data is ready (expect ~565 SecQUE and ~150 FinanceBench examples): ```bash -wc -l nvflow/recipes/finance/datasets/secque/eval.jsonl nvflow/recipes/finance/datasets/financebench/eval.jsonl +wc -l outputs/finance/eval-datasets/secque/eval.jsonl outputs/finance/eval-datasets/financebench/eval.jsonl ``` **prepare_data output:** ``` -nvflow/recipes/finance/datasets/ # shared across workflows +outputs/finance/eval-datasets/ # shared across workflows β”œβ”€β”€ secque/ β”‚ └── eval.jsonl β”œβ”€β”€ financebench/ @@ -156,6 +156,17 @@ Qwen3-4B and GPT-OSS 20B leverage reasoning (thinking mode and Harmony format re Download 10-K and 10-Q filings for 7 demo companies from SEC EDGAR. The download utility is built into nvflow and uses the `edgartools` library to fetch filings and extract sections. +> **Required first:** SEC EDGAR rejects requests that don't identify the caller, and the config ships with placeholders. Edit the `demo` stage in `nvflow/recipes/finance/workflows/download_sec_filings.yaml` before running β€” there is no command-line override: +> +> ```yaml +> stages: +> demo: +> sec_identity_email: your.email@company.com +> sec_identity_company: YourCompany +> ``` +> +> See the [SEC Fair Access Policy](https://www.sec.gov/os/accessing-edgar-data). + **Preview stages:** ```bash uv run nflow list-stages --config nvflow/recipes/finance/workflows/download_sec_filings.yaml @@ -206,6 +217,19 @@ outputs/finance/demo/workflow-2-download-sec/ Generate financial Q&A pairs using the template-based SDG workflow. +> **Required first:** `create_seed_data` reaches both SEC EDGAR and HuggingFace, so before running: +> +> 1. Set your SEC identity in `nvflow/recipes/finance/workflows/sdg/template-based-sdg.yaml` (inherited by the demo config, and shipped with placeholders): +> +> ```yaml +> stages: +> create_seed_data: +> sec_identity_email: your.email@company.com +> sec_identity_company: YourCompany +> ``` +> +> 2. Temporarily clear `HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE` and `TRANSFORMERS_OFFLINE` in your cluster config, since the seed dataset is pulled from HuggingFace. Re-enable them afterwards. See [Offline runtime](troubleshooting.md#offline-runtime). + **Preview stages:** ```bash uv run nflow list-stages --config nvflow/recipes/finance/workflows/sdg/template-based-sdg-demo.yaml @@ -277,7 +301,7 @@ uv run nflow list-stages --config nvflow/recipes/finance/workflows/sft/qwen3_4b. **Pre-check:** If you skipped Step 1 (baseline eval), ensure benchmark datasets exist: ```bash -wc -l nvflow/recipes/finance/datasets/secque/eval.jsonl nvflow/recipes/finance/datasets/financebench/eval.jsonl +wc -l outputs/finance/eval-datasets/secque/eval.jsonl outputs/finance/eval-datasets/financebench/eval.jsonl # Expected: 565 secque + 150 financebench ``` @@ -398,7 +422,7 @@ Stage 7 (`collect_rollouts`) includes automatic sub-jobs: **Pre-check:** If you skipped Step 1 (baseline eval), ensure benchmark datasets exist: ```bash -wc -l nvflow/recipes/finance/datasets/secque/eval.jsonl nvflow/recipes/finance/datasets/financebench/eval.jsonl +wc -l outputs/finance/eval-datasets/secque/eval.jsonl outputs/finance/eval-datasets/financebench/eval.jsonl # Expected: 565 secque + 150 financebench ``` @@ -449,7 +473,7 @@ uv run nflow run collect_rollouts --config nvflow/recipes/finance/workflows/grpo # Post-rollout train/val split (CPU) uv run nflow run train_validation_split --config nvflow/recipes/finance/workflows/grpo/qwen3_4b.yaml -e finance_sec_search -# Training (Megatron, 64 GPUs β€” uses separate config for YaRN + CP=4) +# Training (Megatron, 16 GPUs β€” uses separate config for YaRN + CP=8) uv run nflow run training --config nvflow/recipes/finance/workflows/grpo/qwen3_4b_finsec.yaml -e finance_sec_search ``` @@ -468,7 +492,7 @@ squeue --me # Rollout logs (one per seed per environment) tail -f outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-5-collect-rollouts/*/logs/*.log # Training logs -tail -f outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-7-training/*/grpo-qwen3-4b-*/training-logs/ray-*-job.log +tail -f outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-8-training/*/grpo-qwen3-4b-*/training-logs/ray-*-job.log ``` **Verify rollouts (both environments):** @@ -494,10 +518,10 @@ ls outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-8-training/equivalence_llm ls outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-8-training/finance_sec_search/grpo-qwen3-4b-*/checkpoints/ ``` -**Verify evaluation:** +**Verify evaluation** (results are per-environment, matching the training checkpoints): ```bash -cat outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-8-eval/step-20/eval-results/secque/metrics.json -cat outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-8-eval/step-20/eval-results/financebench/metrics.json +cat outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-9-eval/finance_sec_search/step-20/eval-results/secque/metrics.json +cat outputs/finance/demo/workflow-5-grpo/qwen3_4b/step-9-eval/finance_sec_search/step-20/eval-results/financebench/metrics.json ``` **Output:** @@ -544,10 +568,16 @@ outputs/finance/demo/workflow-5-grpo/ β”‚ β”‚ β”œβ”€β”€ checkpoints/ β”‚ β”‚ └── training-logs/ β”‚ └── step-9-eval/ -β”‚ └── step-20/ -β”‚ └── eval-results/ -β”‚ β”œβ”€β”€ secque/metrics.json -β”‚ └── financebench/metrics.json +β”‚ β”œβ”€β”€ equivalence_llm_judge/ # Per-env results +β”‚ β”‚ └── step-20/ +β”‚ β”‚ └── eval-results/ +β”‚ β”‚ β”œβ”€β”€ secque/metrics.json +β”‚ β”‚ └── financebench/metrics.json +β”‚ └── finance_sec_search/ +β”‚ └── step-20/ +β”‚ └── eval-results/ +β”‚ β”œβ”€β”€ secque/metrics.json +β”‚ └── financebench/metrics.json ``` > **Per-environment training:** Each environment produces a separate model checkpoint. To train a single combined model on both environments, omit `-e` in the training command. @@ -578,50 +608,8 @@ outputs/finance/demo/workflow-5-grpo/ ## Troubleshooting -
-Download fails with "SEC rate limit" - -SEC EDGAR has rate limits. The downloader includes automatic throttling, but if you hit limits: -- Wait 10 minutes and retry -- Ensure `sec_identity_email` is valid in cluster config - -
- -
-"File not found: sec_metadata.parquet" +See the comprehensive **[Finance Recipe Troubleshooting](troubleshooting.md)** guide for issues across all workflows (SEC rate limits, missing `sec_metadata.parquet`, jobs not starting, eval metrics `N/A`, `Address already in use`, offline-runtime errors, resuming interrupted runs, and more). -Download stage may not have completed. Check logs: -```bash -ls outputs/finance/demo/workflow-2-download-sec/download-logs/ -``` - -
- -
-SFT job not starting - -Check SLURM queue and partition availability: -```bash -squeue --me -sinfo -p interactive -``` - -
- -
-Eval metrics show "N/A" - -Ensure the `checkpoint_path` in your SFT/GRPO config's `stages.eval` section matches your actual training output directory. - -
- -
- -vLLM server crashes with "Address already in use" - -Simply re-run the failed stage. The pipeline will retry only the chunks that did not complete. - -
--- [Workflow Documentation](workflows/) | [Stage Reference](stages/) | [Main README](README.md) diff --git a/docs/recipes/finance/stages/document-grounded-sdg.md b/docs/recipes/finance/stages/document-grounded-sdg.md index 48d86d3..9c9a521 100644 --- a/docs/recipes/finance/stages/document-grounded-sdg.md +++ b/docs/recipes/finance/stages/document-grounded-sdg.md @@ -5,18 +5,18 @@ Technical reference for all 7 stages in the document-grounded-sdg workflow. ## Quick Navigation - [dg_sdg_preprocess](#dg_sdg_preprocess) -- [generate_verified_qa](#generate_verified_qa) -- [genselect_answers](#genselect_answers) +- [generate_verified_questions](#generate_verified_questions) +- [generate_answers](#generate_answers) +- [gym_genselect_answers](#gym_genselect_answers) - [evaluate_answers](#evaluate_answers) - [aggregate_answers](#aggregate_answers) -- [difficulty_estimation](#difficulty_estimation) - [dgsdg_post_process](#dgsdg_post_process) --- ## dg_sdg_preprocess -**File:** `nvflow/recipes/finance/stages/sdg/dg_sdg_preprocess.py` +**File:** `nvflow/generic_stage/sdg/document_grounded/dg_sdg_preprocess.py` **Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="dg_sdg_preprocess"` ### Purpose @@ -34,12 +34,12 @@ Converts raw SEC 10-K and 10-Q HTML filings into structured JSONL data for quest | `input_dir` | path | Raw SEC filings directory (10-K and 10-Q HTML files) | Required | | `output_dir` | path | Preprocessed data output directory | Required | | `distribution_dir` | path | Directory with distribution CSVs (SecQue benchmark) | Required | +| `preprocess_module` | str | Dotted module path to domain CLI that chunks + samples | Required | | `max_tokens` | int | Maximum tokens per chunk | 2000 | | `overlap_tokens` | int | Overlap tokens between chunks for context coverage | 100 | | `total_samples` | int | Total samples to generate following distribution | 150000 | | `max_skip_count` | int | Stop sampling after this many skips (non-repeatable) | 20000 | | `seed` | int | Random seed for reproducibility | 42 | -| `preprocess_kwargs` | dict | Additional CPU job settings (partition, etc.) | `{}` | ### Expected Input Structure @@ -85,7 +85,8 @@ ${output_dir}/ dg_sdg_preprocess: input_dir: ${filings_dir}/data output_dir: ${base_data_dir}/step-0-preprocess - distribution_dir: /workspace/nvflow/recipes/finance/workflows/sdg/dg_sdg_distribution + distribution_dir: nvflow/recipes/finance/workflows/sdg/dg_sdg_distribution + preprocess_module: nvflow.recipes.finance.utils.sdg.dg_sdg_data_preprocess max_tokens: 3000 overlap_tokens: 500 total_samples: 150000 @@ -108,67 +109,61 @@ dg_sdg_preprocess: --- -## generate_verified_qa +## generate_verified_questions -**File:** `nvflow/recipes/finance/stages/sdg/document_grounded_question_answer_generation_pipeline.py` -**Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="generate_verified_qa"` +**File:** `nvflow/generic_stage/sdg/document_grounded/generate_verified_questions.py` +**Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="generate_verified_questions"` ### Purpose -Combined stage that generates questions from SEC filing documents, verifies their quality, and generates answers. Executes 6 internal sub-steps. +Q-side of the DG-SDG pipeline. Generates questions from SEC filing documents and verifies their quality. Executes 4 internal sub-steps. ### Internal Sub-Steps -1. **Preprocess Documents** (CPU): Preprocess sampled data for question generation -2. **Generate Questions** (GPU): Create questions from documents -3. **Preprocess Questions** (CPU): Prepare for verification -4. **Verify Questions** (GPU): Verify quality with 5 random seeds -5. **Preprocess Verified** (CPU): Filter by threshold, prepare for answers -6. **Generate Answers** (GPU): Generate answers with 5 random seeds +1. **Q-prep** (CPU): Run the recipe-supplied `question_prep_script` to attach `context` strings to each chunk +2. **Q-gen** (GPU): Generate questions from documents +3. **Q-verify-prep** (CPU): Expand each generated question into N verification trials +4. **Q-verify** (GPU): Per-question Yes/No vote with multiple random seeds ### Inputs | Parameter | Type | Description | |-----------|------|-------------| | `input_folder` | path | Preprocessed JSONL data directory from `dg_sdg_preprocess` (`${base_data_dir}/step-0-preprocess/jsonl/`) | -| `output_dir` | path | Base output directory for all sub-steps | -| `question_preprocess_kwargs` | dict | CPU job settings for preprocessing | +| `output_dir` | path | Q-pipeline output directory (e.g. `${base_data_dir}/step-1-questions`) | +| `question_prep_script` | path | Domain wrapper that injects `context_builder` into `lib.sdg.document_grounded.preprocess.construct_question_generate_input` | +| `gym_path` / `gym_config_paths` / `gym_agent_name` | various | NeMo-Gym defaults; per-substep `question_generation_*` / `question_verify_*` overrides allowed | | `question_generation_kwargs` | dict | GPU settings for question generation | -| `question_verify_kwargs` | dict | GPU settings for verification (5 seeds) | -| `answer_preprocess_kwargs` | dict | CPU settings, includes `threshold` | -| `answer_generation_kwargs` | dict | GPU settings for answer generation (5 seeds) | +| `question_verify_kwargs` | dict | GPU settings for verification (typically 5 seeds) | ### Outputs ``` ${output_dir}/ -β”œβ”€β”€ question_pipeline/ -β”‚ β”œβ”€β”€ generate_input.jsonl # Preprocessed documents -β”‚ β”œβ”€β”€ generated/ # Generated questions -β”‚ β”‚ β”œβ”€β”€ seed_0.jsonl -β”‚ β”‚ └── ... -β”‚ β”œβ”€β”€ verify_input.jsonl # Questions to verify -β”‚ └── verified/ # Verified questions -β”‚ β”œβ”€β”€ seed_0.jsonl -β”‚ └── ... -└── answer_pipeline/ - β”œβ”€β”€ answer_input.jsonl # Verified questions - └── generated/ # Generated answers ← Output - β”œβ”€β”€ seed_0.jsonl - └── ... +β”œβ”€β”€ generate_input.jsonl # step 1 output (q-prep) +β”œβ”€β”€ generated/ # step 2 output (Q-gen rollouts) +β”œβ”€β”€ verify_input.jsonl # step 3 output (q-verify-prep) +└── verified/ # step 4 output (Q-verify rollouts) + # ← consumed by generate_answers ``` ### Configuration Example ```yaml -generate_verified_qa: +generate_verified_questions: input_folder: ${base_data_dir}/step-0-preprocess/jsonl - output_dir: ${base_data_dir}/step-1-qa-pipeline + output_dir: ${base_data_dir}/step-1-questions + dependencies: [dg_sdg_preprocess] + + question_prep_script: nvflow/recipes/finance/utils/sdg/sec_question_prep.py + gym_path: *gym_path + gym_config_paths: *gym_config_paths_format_verification + gym_agent_name: *gym_agent_format_verification question_generation_kwargs: args: model: /models/gpt-oss-120b - server_gpus: 8 + num_gpus: 8 num_chunks: 5 num_random_seeds: 1 ctx_args: >- @@ -178,42 +173,102 @@ generate_verified_qa: question_verify_kwargs: args: model: /models/Qwen3-235B - server_gpus: 8 + num_gpus: 8 num_chunks: 5 num_random_seeds: 5 +``` + +### Resources + +- **Runtime:** ~2-4 hours +- **GPUs:** 40 for question generation, 200 for question verification +- **Models:** GPT-OSS-120B (questions), Qwen3-235B (verification) + +--- + +## generate_answers + +**File:** `nvflow/generic_stage/sdg/document_grounded/generate_answers.py` +**Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="generate_answers"` + +### Purpose + +A-side of the DG-SDG pipeline. Filters questions by verification pass-rate, then generates N candidate answers per surviving question. Executes 2 internal sub-steps. + +### Internal Sub-Steps + +1. **A-prep** (CPU): `construct_answer_generate_input` keeps only questions whose Q-verify pass-rate β‰₯ `threshold` +2. **A-gen** (GPU): Generate answers (typically 5 seeds for downstream genselect) + +### Inputs + +| Parameter | Type | Description | +|-----------|------|-------------| +| `input_dir` | path | Verified-questions directory from `generate_verified_questions` (`${base_data_dir}/step-1-questions/verified`) | +| `output_dir` | path | A-pipeline output directory (e.g. `${base_data_dir}/step-2-answers`) | +| `gym_path` / `gym_config_paths` / `gym_agent_name` | various | NeMo-Gym defaults; per-substep `answer_generation_*` overrides allowed | +| `answer_preprocess_kwargs` | dict | CPU settings, includes `threshold` (Q-verify pass-rate cutoff) | +| `answer_generation_kwargs` | dict | GPU settings for answer generation | + +### Outputs + +``` +${output_dir}/ +β”œβ”€β”€ answer_input.jsonl # step 1 output (a-prep) +└── generated/ # step 2 output (A-gen rollouts; consumed by gym_genselect_answers) + β”œβ”€β”€ output-rs0.jsonl + └── ... +``` + +### Configuration Example + +```yaml +generate_answers: + input_dir: ${base_data_dir}/step-1-questions/verified + output_dir: ${base_data_dir}/step-2-answers + dependencies: [generate_verified_questions] + + gym_path: *gym_path + gym_config_paths: *gym_config_paths_format_verification + gym_agent_name: *gym_agent_format_verification + + answer_preprocess_kwargs: + threshold: 1 answer_generation_kwargs: args: model: /models/gpt-oss-120b - server_gpus: 8 + num_gpus: 8 num_chunks: 5 num_random_seeds: 5 + ctx_args: >- + ++prompt_config=nvflow/recipes/finance/prompts/secque_template.yaml ``` ### Resources -- **Total Runtime:** ~4-8 hours for full pipeline -- **GPUs:** 40 for question generation, 200 for question verification and answer generation -- **Models:** GPT-OSS-120B (questions, answers), Qwen3-235B (verification) +- **Runtime:** ~2-4 hours +- **GPUs:** 200 (5 seeds, 5 chunks each) +- **Model:** GPT-OSS-120B --- -## genselect_answers +## gym_genselect_answers -**File:** `nvflow/recipes/finance/stages/sdg/genselect_answers.py` -**Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="genselect_answers"` +**File:** `nvflow/generic_stage/sdg/document_grounded/gym_genselect_answers.py` +**Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="gym_genselect_answers"` ### Purpose -Select best answer from multiple candidates (same as template-based, but for document-grounded data). +Select the best answer from the multiple candidates produced by `generate_answers` (DG-SDG-specific best-of-N picker that runs through NeMo-Gym). ### Inputs | Parameter | Type | Description | |-----------|------|-------------| -| `input_dir` | path | Answer candidates from generate_verified_qa | +| `input_dir` | path | Answer candidates from `generate_answers` | | `output_file` | path | Selected answers output file | -| `prompt_config` | path | GenSelect prompt | +| `prompt_template` | path | GenSelect prompt | ### Outputs @@ -222,19 +277,19 @@ JSONL file with selected best answers. ### Configuration Example ```yaml - genselect_answers: - input_dir: ${base_data_dir}/step-1-qa-pipeline/answer_pipeline/generated - output_file: ${base_data_dir}/step-2-genselect/selected_answers.jsonl - prompt_config: nvflow/recipes/finance/prompts/genselect_answers.yaml - inline_args: "++inference.tokens_to_generate=16384" - dependencies: [generate_verified_qa] - stage_kwargs: - model: /models/Qwen3-235B-A22B-Instruct-2507 - server_type: vllm - server_gpus: 8 + gym_genselect_answers: + input_dir: ${base_data_dir}/step-2-answers/generated + output_file: ${base_data_dir}/step-3-genselect/selected_answers.jsonl + prompt_template: nvflow/recipes/finance/prompts/genselect_answers.yaml + dependencies: [generate_answers] + + policy_vllm: + model_path: /models/Qwen3-235B-A22B-Instruct-2507 + num_gpus: 8 server_nodes: 1 - num_chunks: 15 - partition: batch + num_chunks: 5 + inference_params: + max_output_tokens: 16384 ``` ### Resources @@ -247,39 +302,38 @@ JSONL file with selected best answers. ## evaluate_answers -**File:** `nvflow/recipes/finance/stages/sdg/evaluate_answers.py` +**File:** `nvflow/generic_stage/sdg/document_grounded/evaluate_answers.py` **Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="evaluate_answers"` ### Purpose -Evaluate answer quality using a large model judge. Runs 5 random seeds for robustness. +Evaluate answer quality using a large model judge. Runs 5 random seeds for robustness. Each seed's judge response ends with a JSON verdict tag `{"answerable": "YES/NO", "correct": "YES/NO"}` (parsed downstream by `aggregate_answers`). ### Inputs | Parameter | Type | Description | |-----------|------|-------------| -| `input_file` | path | Selected answers from genselect_answers | +| `input_file` | path | Selected answers from `gym_genselect_answers` | | `output_dir` | path | Directory for evaluation results | -| `prompt_config` | path | Evaluation prompt | +| `prompt_template` | path | Evaluation prompt | ### Outputs ``` ${output_dir}/ -β”œβ”€β”€ seed_0.jsonl -β”œβ”€β”€ seed_1.jsonl -β”œβ”€β”€ seed_2.jsonl -β”œβ”€β”€ seed_3.jsonl -└── seed_4.jsonl +β”œβ”€β”€ output-rs0.jsonl +β”œβ”€β”€ output-rs1.jsonl +β”œβ”€β”€ output-rs2.jsonl +β”œβ”€β”€ output-rs3.jsonl +└── output-rs4.jsonl ``` -Each file contains evaluation scores: +Each record carries the judge's raw `evaluate_generation`, whose last line is the JSON verdict tag parsed by `aggregate_answers`: ```json { - "question": "...", + "problem": "...", "generation": "...", - "evaluate_generation": "Score: 4.5/5\nReasoning: ...", - "evaluation_score": 4.5 + "evaluate_generation": "...reasoning...\n{\"answerable\": \"YES\", \"correct\": \"YES\"}" } ``` @@ -287,19 +341,21 @@ Each file contains evaluation scores: ```yaml evaluate_answers: - input_file: ${base_data_dir}/step-2-genselect/selected_answers.jsonl - output_dir: ${base_data_dir}/step-3-evaluate - prompt_config: nvflow/recipes/finance/prompts/evaluate_answers.yaml - inline_args: "++generation_key=evaluate_generation ++inference.top_p=0.9 ++inference.temperature=0.8" - dependencies: [genselect_answers] - stage_kwargs: - model: /models/Qwen3-235B-A22B-Instruct-2507 - server_type: vllm - server_gpus: 8 + input_file: ${base_data_dir}/step-3-genselect/selected_answers.jsonl + output_dir: ${base_data_dir}/step-4-evaluate + prompt_template: nvflow/recipes/finance/prompts/evaluate_answers.yaml + generation_key: evaluate_generation + dependencies: [gym_genselect_answers] + + policy_vllm: + model_path: /models/Qwen3-235B-A22B-Instruct-2507 + num_gpus: 8 server_nodes: 1 - num_chunks: 5 - num_random_seeds: 5 - partition: batch + num_chunks: 1 + num_random_seeds: 5 + inference_params: + top_p: 0.9 + temperature: 0.8 ``` ### Resources @@ -312,30 +368,29 @@ Each file contains evaluation scores: ## aggregate_answers -**File:** `nvflow/recipes/finance/stages/sdg/aggregate_answers.py` +**File:** `nvflow/generic_stage/sdg/document_grounded/aggregate_answers.py` **Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="aggregate_answers"` ### Purpose -Aggregate evaluation results from 5 random seeds into final scores. +Aggregate the 5 evaluate seeds: keep a question only if **all** seeds voted `correct=YES` with a consistent `answerable`, and attach the consensus `answerable`. ### Inputs | Parameter | Type | Description | |-----------|------|-------------| -| `input_dir` | path | Evaluation results from evaluate_answers | +| `input_dir` | path | Evaluation results from `evaluate_answers` | | `output_file` | path | Aggregated results output | ### Outputs -JSONL file with aggregated scores: +A single JSONL file of surviving records. The per-seed `evaluate_generation` / `correct` are dropped and a consensus `answerable` is added: ```json { - "question": "...", + "problem": "...", "generation": "...", - "evaluation_scores": [4.5, 4.8, 4.3, 4.7, 4.6], - "mean_score": 4.58, - "std_score": 0.18 + "reference_answer": "...", + "answerable": "YES" } ``` @@ -346,134 +401,32 @@ JSONL file with aggregated scores: --- -## difficulty_estimation - -**File:** `nvflow/recipes/finance/stages/sdg/difficulty_estimation.py` -**Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="difficulty_estimation"` - -### Purpose - -Estimate question difficulty by testing if a small model can answer correctly. Questions the small model fails are considered harder. - -### Two-Step Process - -1. **Small Model Answering**: Qwen3-4B attempts to answer (5 seeds) -2. **Large Model Judging**: GPT-OSS-120B judges if small model succeeded - -### Inputs - -| Parameter | Type | Description | -|-----------|------|-------------| -| `input_file` | path | Aggregated answers | -| `output_file` | path | Answers with difficulty scores | -| `work_dir` | path | Working directory for intermediate files | -| `num_random_seeds` | int | Random seeds for small model (default: 5) | -| `answer_model_kwargs` | dict | Settings for small model (Qwen3-4B) | -| `judge_model_kwargs` | dict | Settings for judge model (GPT-OSS-120B) | - -### Outputs - -JSONL file with difficulty scores: -```json -{ - "question": "...", - "generation": "...", - "difficulty_score": 0, # 0 = hard (small model failed) - "small_model_correct": false, - "small_model_attempts": 5, - "small_model_successes": 0 -} -``` - -**Difficulty Score:** -- `0`: Hard (small model failed all attempts) -- `1-4`: Medium (small model succeeded on some attempts) -- `5`: Easy (small model succeeded on all attempts) - -### Configuration Example - -```yaml - difficulty_estimation: - input_file: ${base_data_dir}/step-4-aggregate/aggregated_answers.jsonl - output_file: ${base_data_dir}/step-5-difficulty-data/answers_with_difficulty.jsonl - work_dir: ${base_data_dir}/step-5-difficulty - num_random_seeds: 5 - dependencies: [aggregate_answers] - - - # Small model for answering (Qwen3-4B) - answer_model_kwargs: - args: - model: /models/qwen34b - server_type: vllm - server_gpus: 8 - server_nodes: 1 - num_chunks: 5 - partition: batch - ctx_args: >- - ++prompt_config=nvflow/recipes/finance/prompts/secque_template.yaml - ++inference.temperature=0.7 - - answer_prompt_config: nvflow/recipes/finance/prompts/secque_template.yaml - - # Large model for judging (GPT-OSS120) - judge_model_kwargs: - args: - model: /models/gpt-oss-120b - server_type: vllm - server_gpus: 8 - server_nodes: 1 - num_chunks: 10 - partition: batch - ctx_args: >- - ++inference.temperature=0.1 - - judge_prompt_config: nvflow/recipes/finance/prompts/judge_difficulty.yaml -``` - -### Resources - -- **GPUs:** 200 (small model), 400 (judge model) -- **Runtime:** 4-8 hours - ---- - ## dgsdg_post_process -**File:** `nvflow/recipes/finance/stages/sdg/document_grounded_data.py` +**File:** `nvflow/generic_stage/sdg/document_grounded/dgsdg_post_process.py` **Registry:** `recipe="finance"`, `workflow="document_grounded_sdg"`, `stage="dgsdg_post_process"` ### Purpose -Clean data and create difficulty-stratified training datasets. +Clean and rename fields, then emit a single `final_result.jsonl` consumed by downstream SFT / GRPO workflows. Records are not split into subsets; the per-stage trim (see `_schemas.py::STAGE_KEEP["dgsdg_post_process"]`) plus the recipe's `domain_keep_fields` defines the final allowlist of fields kept in `final_result.jsonl`. ### Inputs | Parameter | Type | Description | |-----------|------|-------------| -| `input_file` | path | Answers with difficulty from difficulty_estimation | -| `output_dir` | path | Output directory for final datasets | +| `input_file` | path | Aggregated answers from `aggregate_answers` (e.g. `${base_data_dir}/step-5-aggregate/aggregated_answers.jsonl`) | +| `output_dir` | path | Output directory for `final_result.jsonl` | +| `postprocess_script` | path | Domain CLI wrapper around `nvflow.lib.sdg.document_grounded.postprocess.dgsdg_post_process` (e.g. `recipes/finance/utils/sdg/sec_postprocess.py`) | | `seed` | int | Random seed for reproducibility (default: 42) | +| `domain_keep_fields` | list[str] | Recipe-specific fields appended to the generic allowlist before per-stage trim | ### Outputs ``` ${output_dir}/ -β”œβ”€β”€ full_data.jsonl # All cleaned records -β”œβ”€β”€ final_result.jsonl # difficulty_score in [1,2,3,4], filtered -└── hard_rl_data.jsonl # difficulty_score = 0 (hardest) +└── final_result.jsonl # Single cleaned + renamed dataset consumed by SFT / GRPO ``` -**final_result.jsonl** - Medium difficulty training data: -- Medium difficulty questions -- Filtered by filing type and quality -- Ready for training - -**hard_rl_data.jsonl** - Hard difficulty training data: -- Hardest questions (small model failed) -- High-quality answers -- Suitable for advanced training or challenging evaluation - ### Resources - **Compute:** CPU only @@ -485,12 +438,13 @@ ${output_dir}/ | Stage | Purpose | Compute | Runtime | |-------|---------|---------|---------| -| generate_verified_qa | Generate & verify Q&A | GPU | 6-8h | -| genselect_answers | Select best answers | GPU | 1-2h | +| dg_sdg_preprocess | Chunk + sample documents | CPU | 2-4h | +| generate_verified_questions | Generate + verify questions | GPU | 2-4h | +| generate_answers | Generate candidate answers | GPU | 2-4h | +| gym_genselect_answers | Select best answers | GPU | 1-2h | | evaluate_answers | Evaluate quality | GPU | 2-3h | | aggregate_answers | Aggregate scores | CPU | 10m | -| difficulty_estimation | Estimate difficulty | GPU | 2-3h | -| dgsdg_post_process | Create final datasets | CPU | 10m | +| dgsdg_post_process | Clean + rename β†’ final_result.jsonl | CPU | 10m | **Total:** ~10-12 hours for full production run diff --git a/docs/recipes/finance/stages/download-sec.md b/docs/recipes/finance/stages/download-sec.md index e040568..4f016fa 100644 --- a/docs/recipes/finance/stages/download-sec.md +++ b/docs/recipes/finance/stages/download-sec.md @@ -2,12 +2,13 @@ Technical reference for the download-sec workflow stage. -## Stage: sap-500 / demo +## Stage: smoke / demo / sap-500 **File:** `nvflow/recipes/finance/stages/download/download_sec_filings.py` -**Registry:** `recipe="finance"`, `workflow="download-sec"`, `stage="sap-500"` and `stage="demo"` +**Registry:** `recipe="finance"`, `workflow="download-sec"`, `stage="smoke"`, `stage="demo"` and `stage="sap-500"` -> **Note:** Both `sap-500` and `demo` stages use the same implementation but load different configuration files: +> **Note:** All three stages share one implementation and differ only in the ticker config they load: +> - `smoke`: 2 companies, 1 year β€” for pipeline smoke tests > - `demo`: 7 companies (NVDA, AAPL, GOOG, MSFT, CSCO, META, IBM) with 10-K and 10-Q forms (2020-2024) > - `sap-500`: 500+ S&P 500 companies with 10-K, 10-Q, and 8-K forms diff --git a/docs/recipes/finance/stages/eval.md b/docs/recipes/finance/stages/eval.md index 7b5ede7..31e83e4 100644 --- a/docs/recipes/finance/stages/eval.md +++ b/docs/recipes/finance/stages/eval.md @@ -94,12 +94,12 @@ stages: eval: eval_steps: [2600, 5000, 7408] checkpoint_path: ${directories.step-4-training}/model-name - format: megatron # Use "fsdp" for GRPO demo, "megatron" for GRPO production + format: megatron # Match the checkpoint's training backend: "fsdp" or "megatron" baseline_model: /hf_models/Qwen/Qwen3-14B server_type: vllm gpus: 1 inference_args: >- - ++prompt_config=/workspace/nvflow/recipes/finance/prompts/secque_template.yaml + ++prompt_config=nvflow/recipes/finance/prompts/secque_template.yaml ++inference.temperature=0.6 server_args: "--max-model-len 40960 --async-scheduling --reasoning-parser qwen3" ``` @@ -110,7 +110,7 @@ stages: |-----------|------|-------------| | `eval_steps` | list[int] | Training steps to evaluate | | `checkpoint_path` | path | Base path to training checkpoints | -| `format` | str | `"megatron"` (SFT), `"fsdp"` (GRPO demo), or `"megatron"` (GRPO production) | +| `format` | str | Must match the checkpoint's training backend: `"megatron"` (SFT, finance_sec_search GRPO, production) or `"fsdp"` (equivalence_llm_judge GRPO demo); `"hf"` for HF checkpoints | | `baseline_model` | path | HF model path for baseline comparison | | `server_type` | str | Inference server: `"vllm"`, `"openai"` | | `gpus` | int | GPUs for model server | diff --git a/docs/recipes/finance/stages/finance-agent-eval.md b/docs/recipes/finance/stages/finance-agent-eval.md index 7c0d2fd..cd088a1 100644 --- a/docs/recipes/finance/stages/finance-agent-eval.md +++ b/docs/recipes/finance/stages/finance-agent-eval.md @@ -2,14 +2,12 @@ > **Status:** finance_agent evaluation is currently disabled in `eval/base.yaml` pending further validation. The configuration below is preserved for re-enablement. -Technical reference for the finance-agent evaluation stages (vals-ai/finance-agent benchmark). - -> **Note:** Finance agent evaluation is now integrated into the main eval workflow. The `finance_agent` benchmark is defined in `workflows/eval/base.yaml` and runs alongside SEC-QUE and FinanceBench. See [Eval Workflow](../workflows/05-eval.md) for usage. +Technical reference for the finance-agent evaluation stages (vals-ai/finance-agent benchmark). The stage config is defined in `workflows/eval/base.yaml` but is **currently commented out** (see status above); the reference below applies once it is re-enabled. See [Eval Workflow](../workflows/05-eval.md) for the active benchmarks (SEC-QUE, FinanceBench). ## Quick Navigation - [prepare_data](#prepare_data) -- [agent-gpt-oss-120b](#agent-gpt-oss-120b) +- [Agent eval configuration](#agent-eval-configuration) - [Common Agent Parameters](#common-agent-parameters) --- @@ -21,7 +19,7 @@ Technical reference for the finance-agent evaluation stages (vals-ai/finance-age ### Purpose -The shared `prepare_data` stage now downloads **all** benchmark datasets including `finance_agent`. The `finance_agent` dataset is configured in `workflows/eval/base.yaml` under `benchmarks`. +The shared `prepare_data` stage downloads the **enabled** benchmark datasets (`secque`, `financebench`). `finance_agent` is currently excluded from `dataset_names` in `workflows/eval/base.yaml`; re-add it there when the benchmark is re-enabled. ### Finance Agent Dataset @@ -52,13 +50,13 @@ ${output_dir}/ --- -## agent-gpt-oss-120b - -**Registry:** `recipe="finance"`, `workflow="eval"`, `stage="agent-gpt-oss-120b"` +## Agent eval configuration ### Purpose -Evaluate GPT-OSS-120B as a **multi-turn agent** on the finance-agent benchmark. Uses GENERATION_MODULE from the dataset (`agent_gen`) to run the agent loop with tool calls (Tavily web search, SEC EDGAR, HTML parsing). +Evaluate a model as a **multi-turn agent** on the finance-agent benchmark, using the dataset's GENERATION_MODULE (`agent_gen`) to run the agent loop with tool calls (Tavily web search, SEC EDGAR, HTML parsing). + +Eval stages are derived from the `models:` keys in `eval/*.yaml`, so there is no dedicated agent stage to enable β€” you add a model entry. The block below is a worked example using GPT-OSS-120B; it is not shipped in any config. ### Key Differences from Standard Eval @@ -71,12 +69,12 @@ Evaluate GPT-OSS-120B as a **multi-turn agent** on the finance-agent benchmark. | max_turns | N/A | 50 | | max_concurrent_requests | Parallel | 1 (sequential per question) | -### Configuration (from eval/base.yaml benchmarks section) +### Example model entry ```yaml -agent-gpt-oss-120b: +agent-gpt-oss-120b: # example name; choose your own benchmarks: [finance_agent] - datasets_dir: /workspace/nvflow/recipes/finance/datasets + datasets_dir: /workspace/outputs/finance/eval-datasets judge: *judge_finance_strict installation_command: "pip install -q model-library==0.1.8 func-timeout backoff tavily compute-eval @ git+..." extra_args: >- @@ -101,7 +99,7 @@ agent-gpt-oss-120b: ### Resources - **GPUs:** 8 (120B model) -- **Judge:** GPT-5.1 via OpenAI API (external) +- **Judge:** `gpt-5-mini` via OpenAI API (external) - **Tools:** Tavily API (web search), compute-eval for tool execution - **Runtime:** Longer than single-turn (multi-turn + tool calls) @@ -114,7 +112,7 @@ agent-gpt-oss-120b: | Parameter | Description | |-----------|-------------| | `installation_command` | Pip install model-library, func-timeout, tavily, compute-eval | -| `judge` | `judge_finance_strict` (GPT-5.1, sec_judge_strict.yaml) | +| `judge` | `judge_finance_strict` (`gpt-5-mini`, sec_judge_strict.yaml) | | `extra_args.max_turns` | Max agent turns per question (default: 50) | | `extra_args.max_concurrent_requests` | 1 (sequential to avoid API rate limits) | | `rollouts.extra_args.prompt_format` | `openai` (OpenAI function-calling format) | @@ -122,7 +120,7 @@ agent-gpt-oss-120b: ### Judge (judge_finance_strict) Strict finance-domain judge matching vals-ai/finance-agent's judge_new.py: -- **Model:** GPT-5.1 +- **Model:** `gpt-5-mini` - **Prompt:** `sec_judge_strict.yaml` (domain tolerance rules, few-shot examples) - **Temperature:** 0.0 - **Skip extraction:** Yes (judgement only) @@ -156,7 +154,7 @@ models: gpus: 2 nodes: 1 inference_args: >- - ++prompt_config=/workspace/nvflow/recipes/finance/prompts/secque_template.yaml + ++prompt_config=nvflow/recipes/finance/prompts/secque_template.yaml ++inference.tokens_to_generate=32768 ++inference.temperature=0.0 server_args: "--max-model-len 65536 --async-scheduling" @@ -174,7 +172,7 @@ ls outputs/finance/sap-500/workflow-1-baseline-eval/baselines/gpt-oss-120b/eval- cat outputs/finance/sap-500/workflow-1-baseline-eval/baselines/gpt-oss-120b/eval-results/finance_agent/metrics.json | jq . # Check prepared dataset -ls /workspace/nvflow/recipes/finance/datasets/finance_agent/ +ls /workspace/outputs/finance/eval-datasets/finance_agent/ ``` --- @@ -199,4 +197,4 @@ ls /workspace/nvflow/recipes/finance/datasets/finance_agent/ --- -See [Finance Agent Benchmark](../workflows/06-finance-agent-eval.md) for an overview, or [Eval Workflow](../workflows/05-eval.md) for full usage examples and configuration. +See [Eval Workflow](../workflows/05-eval.md) for full usage examples and configuration. diff --git a/docs/recipes/finance/stages/grpo.md b/docs/recipes/finance/stages/grpo.md index f22bd34..a425274 100644 --- a/docs/recipes/finance/stages/grpo.md +++ b/docs/recipes/finance/stages/grpo.md @@ -1,18 +1,54 @@ # GRPO Stages Reference -Technical reference for all 10 stages in the GRPO RL training workflow (9 active + 1 optional). +Technical reference for the GRPO RL training workflow: 10 active stages plus `compute_rewards`, which is optional and commented out by default. + +> **Pass `-e `.** A model config's `environments` block *merges* with `grpo/base.yaml` rather than replacing it, and `base.yaml` declares three environments (`equivalence_llm_judge`, `mcqa`, `finance_sec_search`). Running a single-environment config without `-e` trains all three jointly, including `mcqa`, which is a placeholder with `raw_train_data: null` and is not runnable. ## Quick Navigation -- [data_transformation](#data_transformation) -- [apply_prompt_template](#apply_prompt_template) -- [convert_to_responses_api](#convert_to_responses_api) -- [train_validation_split](#train_validation_split) -- [prepare_data](#prepare_data) -- [collect_rollouts](#collect_rollouts) -- [compute_rewards](#compute_rewards) -- [training](#training) -- [eval](#eval) +Listed in execution order. `prefetch_cache` is optional and has no `step-N` directory. + +- [validate_questions](#validate_questions) β€” step 0 +- [data_transformation](#data_transformation) β€” step 1 +- [apply_prompt_template](#apply_prompt_template) β€” step 2 +- [convert_to_responses_api](#convert_to_responses_api) β€” step 3 +- [prepare_data](#prepare_data) β€” step 4 +- [prefetch_cache](#prefetch_cache) β€” optional +- [collect_rollouts](#collect_rollouts) β€” step 5 +- [compute_rewards](#compute_rewards) β€” step 6, optional +- [train_validation_split](#train_validation_split) β€” step 7 +- [training](#training) β€” step 8 +- [eval](#eval) β€” step 9 + +--- + +## validate_questions + +**File:** `nvflow/recipes/finance/stages/rl/validate_questions.py` +**Registry:** `recipe="finance"`, `workflow="grpo"`, `stage="validate_questions"` + +### Purpose + +Drop structurally-broken SDG questions before they enter the pipeline, per environment, in two phases: + +1. **Regex prefilter (CPU).** Drops questions that say "the company" / "the firm" with no named company or ticker anywhere in the text. Deliberately narrow β€” recall over precision. +2. **LLM classifier (GPU).** Asks a judge model (GPT-OSS-120B by default) for `VALID` / `INVALID` on each survivor. Parse failures default to `VALID`. + +The kept stream is written where `data_transformation` can read it, so a model config re-points `env.raw_train_data` at this stage's output. + +### Outputs + +``` +${step-0-validate-questions}/${env_name}/ +β”œβ”€β”€ final_result.jsonl # VALID records, consumed by data_transformation +β”œβ”€β”€ phase1_regex/ # prefiltered + dropped + stats (audit) +└── phase2_llm/ # raw generation, parsed tags, dropped, stats +``` + +### Resources + +- **Phase 1:** CPU only +- **Phase 2:** GPU, for the judge model --- @@ -154,7 +190,7 @@ Split data into training and validation sets using stratified sampling to mainta ### Purpose -Run `ng_prepare_data` to stamp each JSONL record with an `agent_ref` field that tells NeMo-Gym which agent server to route the example to during training. Auto-generates an agent config overlay YAML from the workflow's `agents` list. +Run `gym dataset collate` (formerly `ng_prepare_data`) to stamp each JSONL record with an `agent_ref` field that tells NeMo-Gym which agent server to route the example to during training. Auto-generates an agent config overlay YAML from the workflow's `agents` list. ### Inputs @@ -171,7 +207,7 @@ Run `ng_prepare_data` to stamp each JSONL record with an `agent_ref` field that ### Modes -- **`train_preparation`**: Produces `train.jsonl` + `validation.jsonl` +- **`train_preparation`**: Produces `train.jsonl`. Only a single `train` dataset is collated here; the train/validation split happens later, in [train_validation_split](#train_validation_split), on reward-filtered data. ### Agent Configuration @@ -192,13 +228,13 @@ agents: - name: train type: train license: "TBD" - jsonl_fpath: ${directories.step-3-train-validation-split}/train.jsonl + jsonl_fpath: ${directories.step-3-convert-to-responses-api}/train.jsonl ``` ### Outputs - `${output_dir}/agent_config_overlay.yaml` β€” Auto-generated agent config -- `${output_dir}/train.jsonl` + `validation.jsonl` β€” with `agent_ref` routing fields +- `${output_dir}/train.jsonl` β€” with `agent_ref` routing fields ### Resources @@ -207,6 +243,39 @@ agents: --- +## prefetch_cache + +**File:** `nvflow/recipes/finance/stages/rl/prefetch_cache.py` +**Registry:** `recipe="finance"`, `workflow="grpo"`, `stage="prefetch_cache"` + +### Purpose + +Optional CPU-only stage that populates the SEC filing metadata cache before rollout collection. Doing it here keeps SEC.gov calls out of the GPU-intensive rollout jobs and avoids races when several seeds share one cache directory. + +It runs per environment and processes only those whose config carries a `prefetch` block; the rest are skipped silently. In practice that means `finance_sec_search`. + +### Inputs + +Read from each environment's `prefetch` block: + +| Key | Description | +|-----|-------------| +| `script` | Upstream Gym prefetch script to run | +| `cache_dir` | Where the cache is written | +| `ticker_config` | Ticker set to prefetch | +| `force` | Re-fetch even if the cache is populated (default `false`) | + +### Outputs + +The cache directory declared by the environment. For `finance_sec_search` this is `cache-finance-sec-search`, i.e. `${base_output_dir}/cache/finance_sec_search`, holding `filings/`, `filings_metadata/` and `tickers.json`. + +### Resources + +- **Compute:** CPU only +- **Network:** needs SEC EDGAR access, so run it on a connected node + +--- + ## collect_rollouts **File:** `nvflow/recipes/finance/stages/rl/collect_rollouts.py` @@ -218,38 +287,57 @@ Collect model rollouts against a NeMo-Gym environment with reward scoring. Suppo ### Inputs +Top-level keys are orchestration; rollout behaviour is nested under `rollout`. + | Parameter | Type | Description | Default | |-----------|------|-------------|---------| | `output_dir` | path | Output directory | Required | -| `gym_path` | path | Path to NeMo-Gym | Required | -| `container` | string | Container name | Required | -| `input_data` | path | Prepared JSONL from prepare_data | Required | -| `agent_name` | string | Agent name (must match prepare_data) | Required | -| `model_path` | path | Model to collect rollouts from | Required | -| `nemo_gym_config_paths` | list | NeMo-Gym config paths | Required | -| `num_repeats` | int | Repeats per sample | `1` | -| `num_samples_in_parallel` | int | Concurrent requests | `4` | +| `prepare_data_dir` | path | Collated data from `prepare_data` | Required | +| `gym_path` | path | NeMo-Gym root inside the container | `/opt/Gym` | +| `gym_uv_venv_dir` | path | Baked per-component venvs reused by `ng_run` | `/opt/gym-venvs` | +| `container` | string | Rollout client + Gym env servers (CPU) | `nemo-gym` | +| `postprocess_container` | string | Merge/analyze/aggregate/filter (CPU) | `nemo-skills` | +| `vllm_container` | string | Policy and judge vLLM servers (GPU) | `vllm-grpo` | +| `environments` | dict | Environments to collect for | `${environments}` | + +**`rollout`** β€” job fan-out and per-request settings: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `num_samples_in_parallel` | int | Concurrent requests | `64` | +| `max_num_samples` | int | Truncate to first N rows; `null` for all | `null` | | `num_chunks` | int | Split input into N parallel jobs | `1` | -| `num_random_seeds` | int | Independent runs per chunk | `1` | +| `num_random_seeds` | int | Independent runs per chunk | `8` | | `starting_seed` | int | First seed value | `0` | -| `dependent_jobs` | int | Chain N+1 Slurm jobs per chunk via `afterany` for timeout recovery | `0` | -| `responses_create_params` | dict | Pass-through params for NeMo-Gym (e.g., `max_output_tokens`) | `{}` | +| `dependent_jobs` | int | Chain N+1 jobs per (seed, chunk) for timeout resume | `0` | | `rerun_done` | bool | Force re-execution | `false` | -| `num_gpus` | int | GPUs per Slurm job | `8` | -| `tensor_parallel_size` | int | Policy vLLM TP | `2` | +| `responses_create_params` | dict | Per-request overrides, e.g. `max_output_tokens` | `{}` | + +**`rollout.policy_vllm`** β€” the policy server, shared across environments. `num_gpus`, `server_nodes`, `base_url` and `model_path` are orchestration-only; every other key becomes a `--key value` argument to `vllm serve`. + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `model_path` | path | Model to serve | Required, set in the model config | +| `num_gpus` | int | Slurm GPUs for this endpoint; `0` with `base_url` for an external server | `2` | +| `server_nodes` | int | Nodes for this vLLM; `>1` uses Ray | `1` | | `max_model_len` | int | Max sequence length | `32768` | -| `vllm_base_url` | string | External vLLM URL (optional) | None | +| `enable_auto_tool_choice` | bool | Required for tool-calling environments | `true` | +| `tool_call_parser` | string | Tool-call parser | `hermes` | + +> **Don't set `tensor_parallel_size`.** It is derived from `num_gpus` and is silently ignored here. ### Judge Configuration -| Parameter | Type | Description | -|-----------|------|-------------| -| `judge_model_path` | path | Local vLLM judge model | -| `judge_tensor_parallel_size` | int | Judge TP size | -| `judge_max_model_len` | int | Judge max sequence length | -| `judge_openai_base_url` | string | External OpenAI API URL | -| `judge_openai_model` | string | OpenAI model name | -| `judge_openai_api_key` | string | API key override (defaults to `$OPENAI_API_KEY`) | +The judge is configured **per environment**, not on the stage, because each environment decides whether it needs one: + +```yaml +environments: + finance_sec_search: + judge_vllm: + num_gpus: 0 # 0 means no local judge -- override in the model config +``` + +Set `num_gpus` above zero to stand up a local judge vLLM for that environment, and use `responses_create_params` alongside it to override the shared rollout defaults. ### Execution Model @@ -419,7 +507,7 @@ The stage validates parallelism before job submission: ### Outputs ``` -${output_dir}/grpo-{model}-{nodes}n-tp{tp}-cp{cp}-seq{seq}k/ +${output_dir}/grpo-{model}-{total_gpus}g-tp{tp}-cp{cp}-seq{seq}k/ β”œβ”€β”€ checkpoints/ β”‚ β”œβ”€β”€ step_1/ β”‚ └── step_2/ @@ -427,12 +515,14 @@ ${output_dir}/grpo-{model}-{nodes}n-tp{tp}-cp{cp}-seq{seq}k/ └── run_metadata_*.yaml # Full config for reproducibility ``` +The directory name is built from the resolved layout, so `grpo-qwen3-4b-16g-tp2-cp1-seq32k` means 16 GPUs total, TP=2, CP=1 and a 32K sequence budget. + ### Resources | Model Size | GPUs | Runtime (demo) | |------------|------|----------------| -| 4B | 16 (2 nodes) | ~20 min | -| 14B | 64 (8 nodes) | TBD | +| 4B | 16 | ~20 min | +| 30B-A3B | 64 | Longer; see `grpo/qwen3_30b_a3b.yaml` | --- @@ -454,7 +544,7 @@ Also registered for the SFT workflow, making it a shared evaluation stage across | `eval_output_dir` | path | Output directory for evaluation results | Required | | `eval_steps` | list | Checkpoint steps to evaluate | `[]` | | `checkpoint_path` | path | Path to training checkpoints | Required | -| `format` | string | Checkpoint format: `"hf"`, `"fsdp"`, `"megatron"` | `"fsdp"` (demo) / `"megatron"` (production) | +| `format` | string | Checkpoint format; match the training backend: `"hf"`, `"fsdp"` (equivalence demo), `"megatron"` (finance_sec_search demo + production) | backend-dependent | | `baseline_model` | path | Baseline model for comparison evaluation | Optional | | `server_type` | string | Inference server type | `"vllm"` | | `gpus` | int | GPUs for inference server | `1` | diff --git a/docs/recipes/finance/stages/sft.md b/docs/recipes/finance/stages/sft.md index 312d5fe..ef95ca9 100644 --- a/docs/recipes/finance/stages/sft.md +++ b/docs/recipes/finance/stages/sft.md @@ -204,7 +204,7 @@ Group training examples by total sequence length (input + output tokens) to redu | `input_file` | path | Training data from train_validation_split | Required | | `output_dir` | path | Directory for grouped/bucketed data | Required | | `tokenizer_path` | path | Tokenizer for computing lengths (optional if pre-computed) | None | -| `bucket_sizes` | list | Token length boundaries for buckets | `[16000, 32000, 64000]` | +| `bucket_sizes` | list | Token length boundaries for buckets | `[16000, 24000, 32000, 48000]` | ### Bucket Configuration @@ -255,43 +255,68 @@ Fine-tune the language model on financial Q&A data using supervised learning. ### Inputs +Training uses NeMo-RL's config schema: pick a `preset`, then patch it through `overrides`, which is passed to NeMo-RL nested and unflattened. + | Parameter | Type | Description | |-----------|------|-------------| -| `model_name_or_path` | path | Base model to fine-tune | -| `train_file` | path | Training data | -| `val_file` | path | Validation data | -| `output_dir` | path | Directory for checkpoints and logs | -| `num_train_epochs` | int | Number of training epochs (default: 3) | -| `learning_rate` | float | Learning rate (default: 2e-5) | -| `per_device_train_batch_size` | int | Batch size per GPU (default: 4) | -| `gradient_accumulation_steps` | int | Gradient accumulation (default: 8) | -| `save_steps` | int | Checkpoint save frequency (default: 500) | -| `eval_steps` | int | Evaluation frequency (default: 500) | +| `model_name` | string | Model identifier, e.g. `Qwen/Qwen3-14B` | +| `hf_checkpoint_path` | path | Base model on disk, e.g. `/hf_models/Qwen/Qwen3-14B` | +| `backend` | string | `megatron` or `dtensor` | +| `total_gpus` | int | GPUs for the job; data parallelism is derived from it | +| `dependent_jobs` | int | Extra chained jobs, for training longer than one time limit | +| `preset` | string | Base config to start from, e.g. `sft-base` | +| `overrides` | dict | Nested patch over the preset, grouped into `sft`, `checkpointing`, `policy` and `data` | + +Commonly overridden keys: + +| Key | Description | +|-----|-------------| +| `sft.max_num_epochs` | Number of epochs | +| `sft.val_period` | Validate every N steps | +| `checkpointing.save_period` | Save every N steps | +| `checkpointing.keep_top_k` | Checkpoints to retain | +| `policy.train_global_batch_size` | Global batch size | +| `policy.train_micro_batch_size` | Per-rank micro batch | +| `policy.max_total_sequence_length` | Sequence budget | +| `policy.megatron_cfg.*` | Parallelism (`tensor_model_parallel_size`, `context_parallel_size`, …) | +| `policy.megatron_cfg.optimizer.lr` | Learning rate | ### Training Configuration ```yaml -training: - learning_rate: 2e-5 - global_batch_size: 128 - max_num_epochs: 5 +stages: + training: + model_name: Qwen/Qwen3-14B + hf_checkpoint_path: /hf_models/Qwen/Qwen3-14B + backend: megatron + total_gpus: 256 + preset: "sft-base" + overrides: + sft: + max_num_epochs: 3 + policy: + train_global_batch_size: 128 + max_total_sequence_length: 49152 + megatron_cfg: + tensor_model_parallel_size: 4 + context_parallel_size: 8 + optimizer: + lr: 5e-6 ``` ### Outputs ``` -${output_dir}/ +${output_dir}/model-{model}-{total_gpus}g-tp{tp}-pp{pp}-cp{cp}-seq{seq}k/ β”œβ”€β”€ checkpoints/ -β”‚ β”œβ”€β”€ checkpoint-500/ -β”‚ β”œβ”€β”€ checkpoint-1000/ -β”‚ β”œβ”€β”€ checkpoint-1500/ -β”‚ └── final/ # ← Final model -β”œβ”€β”€ logs/ -β”‚ └── training.log -β”œβ”€β”€ runs/ # Tensorboard logs -└── training_args.json +β”‚ β”œβ”€β”€ step_10/ +β”‚ └── step_20/ +β”œβ”€β”€ training-logs/ +└── run_metadata_*.yaml ``` +Checkpoints are step-numbered; there is no `final/` directory. The `eval` stage converts a chosen step to HuggingFace format when it needs one. + ### Resources | Model Size | GPUs | Memory/GPU | Runtime | @@ -401,24 +426,20 @@ Convert Qwen3 chat-templated training data to OpenAI messages format. Parses Qwe ## Common Training Parameters +All of these live under `overrides` in the training stage. + ### Learning Rate -| Model Size | Recommended LR | -|------------|----------------| -| 7-14B | 2e-5 | -| 32B | 1e-5 | -| 70B+ | 5e-6 | +Set at `policy.megatron_cfg.optimizer.lr`. The shipped configs use `5e-6` with `min_lr: 5e-7`, cosine decay, and warmup from `1e-7`. Treat `5e-6` as the starting point rather than scaling by model size. ### Batch Size -Effective batch size = `per_device_train_batch_size` Γ— `gradient_accumulation_steps` Γ— `total_gpus` - -Recommended: 32-128 for most models +`policy.train_global_batch_size` is the global batch, and `policy.train_micro_batch_size` the per-rank micro batch; gradient accumulation is derived from the two together with the data-parallel width. The production 14B config uses `128` global and `1` micro. ### Checkpointing -- **save_steps**: 500-1000 (more frequent for smaller datasets) -- **save_total_limit**: 3-5 (keep only recent checkpoints to save space) -- **eval_steps**: Same as save_steps +- **`checkpointing.save_period`**: save every N steps β€” `100` for full training, `10` in the demo +- **`checkpointing.keep_top_k`**: checkpoints to retain +- **`sft.val_period`**: validate every N steps See [SFT Workflow](../workflows/04-sft.md) for usage examples and configuration details. diff --git a/docs/recipes/finance/troubleshooting.md b/docs/recipes/finance/troubleshooting.md index 3e53e9e..8da6307 100644 --- a/docs/recipes/finance/troubleshooting.md +++ b/docs/recipes/finance/troubleshooting.md @@ -5,11 +5,14 @@ Comprehensive troubleshooting guide for common issues across all finance recipe ## Quick Navigation - [Cluster & Infrastructure](#cluster--infrastructure) -- [Offline Runtime](#self-sufficient-runtime) +- [Offline Runtime](#offline-runtime) - [Resource Issues](#resource-issues) - [Data Issues](#data-issues) - [Training Issues](#training-issues) - [Workflow-Specific Issues](#workflow-specific-issues) +- [Resuming Interrupted Workflows](#resuming-interrupted-workflows) +- [Frequently Asked Questions](#frequently-asked-questions) +- [Getting Additional Help](#getting-additional-help) --- @@ -92,59 +95,62 @@ scontrol show config | grep SLURM_VERSION # Confirmed: SLURM 25.11.2 needs this fix, SLURM 24.x works without it ``` -**Additional notes:** -- If Ray cluster hangs during initialization, apply this fix -- The fix changes how containers are executed (uses `enroot exec` instead of `--container-name`) -- Test on your cluster - symptom is Ray cluster initialization hang +This changes how containers are launched, using `enroot exec` instead of `--container-name`. --- ## Offline Runtime -The default NVFlow images (`nvflow-nemo-rl`, `nvflow-nemo-skills`, `nvflow-vllm`, `nvflow-vllm-grpo`) are built to run with **no outbound network access** at job time. Most "weird" runtime errors on a freshly-deployed cluster trace back to a missing offline asset, a stale overlay mount, or an env var that was cleared. +The NVFlow images (`nvflow-nemo-skills`, `nvflow-vllm` at both tags, `nvflow-nemo-gym`, `nvflow-nemo-rl`) run with **no outbound network access** at job time, including the `training` stage: `nvflow-nemo-rl` bakes the Gym venvs at build time, so nothing needs resolving over the network. `UV_OFFLINE` is nonetheless left **unset**, which preserves dev mode: mount local Gym source and `uv` resolves it. Most "weird" runtime errors on a freshly-deployed cluster trace back to a missing offline asset, a stale overlay mount, or an env var that was cleared. For the full build / deploy / verify flow, see [INSTALL.md](../../../INSTALL.md) and [`dockerfiles/docker_instructions.md`](../../../dockerfiles/docker_instructions.md). -### GRPO `installation_command` fails with `No such file or directory` +### `huggingface_hub.errors.OfflineModeIsEnabled` / `LocalEntryNotFoundError` -**Problem:** A GRPO stage (`prepare_data`, `collect_rollouts`, `compute_rewards`, or `training`) fails immediately after `source /opt/NeMo-RL/3rdparty/Gym-workspace/Gym/.venv/bin/activate` with: +**Problem:** A stage fails trying to pull a model or dataset from HuggingFace Hub. -``` -bash: /opt/NeMo-RL/3rdparty/Gym-workspace/Gym/.venv/bin/activate: No such file or directory -``` +**Cause:** Air-gap mode is on (`HF_HUB_OFFLINE=1`, etc.) but the asset isn't pre-staged on disk. -**Cause:** You bind-mounted a host clone of NeMo-RL or NeMo-Gym at `/opt/NeMo-RL` (or `/opt/NeMo-RL/3rdparty/Gym-workspace/Gym`), which shadows the baked `.venv` inside the `nvflow-nemo-rl` image. +**Solution:** +- **Models:** Pre-download to your mounted `hf_models` directory with `hf download` -- see [INSTALL.md β†’ Download Models](../../../INSTALL.md#download-models). +- **Datasets / SEC filings:** Some stages (`download_sec_filings`, `create_seed_data`, eval `prepare_data`, GRPO `prepare_data` with `should_download: true`) need internet on first run. Run them on a connected node with the three `HF_*_OFFLINE` flags **temporarily commented out** in `my_cluster.yaml`. The artifacts persist under `/workspace` and are reused by every subsequent run. -**Solution:** Remove the overlay mounts from `cluster_configs/my_cluster.yaml`. The self-sufficient image already contains everything GRPO needs: +### GRPO `training` fails: `uv` tries to resolve, or `ng_run` / `nemo_gym` not found -```yaml -mounts: - # COMMENT THESE OUT (or delete) for normal production runs: - # - :/opt/NeMo-RL - # - :/opt/NeMo-RL/3rdparty/Gym-workspace/Gym -``` +**Problem:** The `training` stage fails soon after start with `uv` trying to download packages, a hung resolution, or a missing Gym module. -See [INSTALL.md β†’ Setup NeMo-RL & NeMo-Gym Sources](../../../INSTALL.md#setup-nemo-rl--nemo-gym-sources-for-grpo) for when (rarely) the overlay is correct. +**Cause:** Nothing should resolve at runtime β€” `nvflow-nemo-rl` bakes one Gym venv per component. A resolve attempt means those baked venvs aren't the ones in use, which has two usual causes: a host clone bind-mounted over `/opt/nemo-rl/3rdparty/Gym-workspace/Gym`, shadowing the baked source and venvs; or the job running the stock upstream `nemo-rl` base, which ships the RL environment but leaves the Gym venvs unbuilt. -### `huggingface_hub.errors.OfflineModeIsEnabled` / `LocalEntryNotFoundError` +**Solution:** +1. Confirm `containers.nemo-rl` in your cluster config points at the image built from [`dockerfiles/Dockerfile.nemo-rl`](../../../dockerfiles/Dockerfile.nemo-rl), not the stock base. +2. Remove any Gym or NeMo-RL source mount from the `mounts:` block. +3. Only if you are deliberately running dev mode against mounted source: leave `UV_OFFLINE` unset and confirm the compute nodes can reach a pypi mirror. See [`docs/development/nemo-rl-gym.md`](../../development/nemo-rl-gym.md). -**Problem:** A stage fails trying to pull a model or dataset from HuggingFace Hub. +The Gym-only stages (`collect_rollouts`, `compute_rewards`, `prefetch_cache`, `prepare_data`) instead run on the self-contained `nvflow-nemo-gym` image (baked venvs, no build); if one of those reports `ng_run: command not found`, the image is missing its baked venvs -- re-check the nemo-gym build in [`docs/maintainers/containers.md`](../../maintainers/containers.md). -**Cause:** Air-gap mode is on (`HF_HUB_OFFLINE=1`, etc.) but the asset isn't pre-staged on disk. +### `omegaconf.errors.InterpolationKeyError: Interpolation key '' not found` after mounting a Gym branch -**Solution:** -- **Models:** Pre-download to your mounted `hf_models` directory with `hf download` -- see [INSTALL.md β†’ Download Models](../../../INSTALL.md#download-models). -- **Datasets / SEC filings:** Some stages (`download_sec_filings`, `create_seed_data`, eval `prepare_data`, GRPO `prepare_data` with `should_download: true`) need internet on first run. Run them on a connected node with the three `HF_*_OFFLINE` flags **temporarily commented out** in `my_cluster.yaml`; keep `UV_OFFLINE=true` set. The artifacts persist under `/workspace` and are reused by every subsequent run. +**Problem:** A `training` or `ng_run`-driven job fails at NeMo-Gym config-load time with, e.g.: -### `uv` errors with "package not installed" or tries to resolve from PyPI +``` +omegaconf.errors.InterpolationKeyError: Interpolation key 'tavily_api_key' not found + full_key: tavily_api_key + object_type=dict +``` + +**Cause:** The Gym source introduced a new `${}` interpolation in a resource-server YAML that the overlays under `nvflow/recipes/finance/workflows/grpo/overlays/` don't yet define. This is drift between the Gym source and the overlays, not a runtime requirement β€” the runtime treats the value as optional (an empty `tavily_api_key` disables Tavily web_search gracefully). -**Problem:** A Ray worker or stage script fails because `uv` is trying to download a package. +**Solution (clean, no upstream change):** Add a placeholder for the missing key in the relevant overlay under `nvflow/recipes/finance/workflows/grpo/overlays/`. For `tavily_api_key` specifically, that's `finance_sec_search_env.yaml`: -**Cause (usual):** Someone enabled `NRL_FORCE_REBUILD_VENVS=true` in offline mode. That flag forces Ray workers to re-resolve packages via `uv`, which requires internet. +```yaml +# Required since upstream Gym introduced ${tavily_api_key} in finance_sec_search.yaml. +# Empty string disables tavily gracefully -- finance_sec_search uses SEC tools only. +tavily_api_key: "" +``` -**Solution:** Comment out `NRL_FORCE_REBUILD_VENVS` in `my_cluster.yaml`. It's only safe to enable on a connected node when you've bind-mounted a host NeMo-RL source overlay and changed the source tree -- see [`docs/cluster-configuration.md`](../../cluster-configuration.md#nemo-rl--grpo-variables-dev-mode-only). +Restart the job; OmegaConf will resolve the interpolation against the overlay value and the resource server will log `No tavily_api_key configured β€” web_search will be unavailable` and continue. -**Cause (rare):** A baked venv is genuinely missing a dependency. Rebuild the image with the missing package added to the Dockerfile and re-run the sanity checks from [`dockerfiles/docker_instructions.md` Β§2](../../../dockerfiles/docker_instructions.md#2-sanity-checks-blockers). +If this happens for a key other than `tavily_api_key`, the same recipe applies: identify which Gym resource-server YAML references the new `${}`, and add the corresponding overlay placeholder under `nvflow/recipes/finance/workflows/grpo/overlays/`. ### `tiktoken` / `openai_harmony` fails to load offline @@ -163,7 +169,7 @@ env_vars: Verify the cache exists inside the image: ```bash -docker run --rm nvflow-nemo-skills:0229040 ls /opt/tiktoken_cache +docker run --rm nvflow-nemo-skills:v1.1.2 ls /opt/tiktoken_cache # Expect: cl100k_base.tiktoken (and o200k_base.tiktoken in vllm images) ``` @@ -175,7 +181,7 @@ docker run --rm nvflow-nemo-skills:0229040 ls /opt/tiktoken_cache **Solution:** Already fixed in `Dockerfile.nemo-skills` (apt `tzdata`). If you see this in a custom-built image, confirm `tzdata` is installed: ```bash -docker run --rm nvflow-nemo-skills:0229040 bash -c \ +docker run --rm nvflow-nemo-skills:v1.1.2 bash -c \ 'python3 -c "import pyarrow as pa; pa.array([], type=pa.timestamp(\"ns\", tz=\"UTC\")); print(\"OK\")"' ``` diff --git a/docs/recipes/finance/workflows/02-template-based-sdg.md b/docs/recipes/finance/workflows/02-template-based-sdg.md index a41d8dc..8c6729a 100644 --- a/docs/recipes/finance/workflows/02-template-based-sdg.md +++ b/docs/recipes/finance/workflows/02-template-based-sdg.md @@ -18,6 +18,7 @@ Before running this workflow, ensure you have: - **Why needed:** Stage 0 downloads the [SecQue dataset](https://huggingface.co/datasets/nvidia/SecQue) (seed questions) from HuggingFace - **Public dataset:** No token required for public access, but token avoids rate limits - **Login alternative:** Run `huggingface-cli login` if you prefer interactive login + - **Offline clusters:** Because Stage 0 reaches the Hub, temporarily clear `HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE` and `TRANSFORMERS_OFFLINE` in your cluster config for this run, then restore them. See [Offline runtime](../troubleshooting.md#offline-runtime) - βœ… **SEC EDGAR identity configured** in workflow YAML: ```yaml diff --git a/docs/recipes/finance/workflows/03-document-grounded-sdg.md b/docs/recipes/finance/workflows/03-document-grounded-sdg.md index 3e0e621..7afd9b8 100644 --- a/docs/recipes/finance/workflows/03-document-grounded-sdg.md +++ b/docs/recipes/finance/workflows/03-document-grounded-sdg.md @@ -2,13 +2,13 @@ ## Purpose -Generate high-quality financial Q&A pairs directly from SEC filing documents with built-in verification, evaluation, and difficulty estimation. +Generate high-quality financial Q&A pairs directly from SEC filing documents with built-in question verification, multi-seed answer evaluation, and per-stage field trimming. -> **Note:** This workflow generates ~800K Q&A pairs. SFT integration is currently in progress. For production SFT pipeline, see [Template-Based SDG](02-template-based-sdg.md). +> **Note:** This workflow generates ~800K Q&A pairs in a single `final_result.jsonl`. The previous difficulty-stratified outputs (`full_data.jsonl`, `hard_rl_data.jsonl`) and the `difficulty_estimation` stage have been removed; downstream SFT / GRPO workflows read `final_result.jsonl` directly. For the production template-based pipeline, see [Template-Based SDG](02-template-based-sdg.md). ## Prerequisites -- βœ… SEC filings downloaded ([Workflow 1](01-download-sec.md)) +- SEC filings downloaded ([Workflow 1](01-download-sec.md)) - Will be preprocessed in Stage 0 (dg_sdg_preprocess) ## Key Differences from Template-Based @@ -18,55 +18,54 @@ Generate high-quality financial Q&A pairs directly from SEC filing documents wit | **Question Source** | Seed questions | Generated from documents | | **Verification** | None | Built-in verification step | | **Quality Control** | GenSelect + Filter | GenSelect + Evaluation + Aggregation | -| **Difficulty** | Not estimated | Estimated via small model testing | -| **Output** | Single dataset | Stratified by difficulty (medium/hard) | +| **Output** | Single dataset | Single `final_result.jsonl` (no stratification) | ## Pipeline Flow ``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 0. dg_sdg_preprocess β”‚ Preprocessing: SEC HTML β†’ Chunked JSONL -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 1. generate_verified_qa β”‚ Q&A Generation: Questions + Answers -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 2. genselect_answers β”‚ Selection: Best answer from candidates -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 3. evaluate_answers β”‚ Evaluation: Quality scoring (5 seeds) -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 4. aggregate_answers β”‚ Aggregation: Combine evaluation results -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 5. difficulty_estimationβ”‚ Difficulty: Small model testing -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 6. dgsdg_post_process β”‚ Output: Stratified training datasets -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 0. dg_sdg_preprocess β”‚ Preprocessing: SEC HTML β†’ Chunked JSONL +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 1. generate_verified_questionsβ”‚ Q-pipeline: prep + Q-gen + verify-prep + Q-verify +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 2. generate_answers β”‚ A-pipeline: a-prep (threshold filter) + A-gen +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 3. gym_genselect_answers β”‚ Selection: Best answer from candidates +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 4. evaluate_answers β”‚ Evaluation: Quality scoring (multi-seed) +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 5. aggregate_answers β”‚ Aggregation: Combine evaluation results +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ 6. dgsdg_post_process β”‚ Output: Cleaned + renamed β†’ final_result.jsonl +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` ## 7 Stages (Overview) 0. **dg_sdg_preprocess**: Preprocess SEC filings (chunk HTML β†’ create JSONL data following SecQue distribution) -1. **generate_verified_qa**: Generate questions from documents, verify them, generate answers (6 internal sub-steps) -2. **genselect_answers**: Select best answer from multiple candidates -3. **evaluate_answers**: Evaluate answer quality (5 random seeds for robustness) -4. **aggregate_answers**: Aggregate evaluation results -5. **difficulty_estimation**: Estimate difficulty using small model -6. **dgsdg_post_process**: Clean and create difficulty-stratified datasets +1. **generate_verified_questions**: Generate questions from documents and verify them (4 internal sub-steps: q-prep + Q-gen + verify-prep + Q-verify) +2. **generate_answers**: Filter questions by verification pass-rate, generate N candidate answers (2 internal sub-steps: a-prep + A-gen) +3. **gym_genselect_answers**: Select best answer from multiple candidates +4. **evaluate_answers**: Evaluate answer quality (multi-seed for robustness) +5. **aggregate_answers**: Aggregate evaluation results +6. **dgsdg_post_process**: Clean + rename fields, emit single `final_result.jsonl` consumed by downstream SFT / GRPO **See [technical reference](../stages/document-grounded-sdg.md) for detailed stage documentation.** @@ -92,21 +91,21 @@ uv run nflow run-all --config nvflow/recipes/finance/workflows/sdg/document-grou # Stage 0: Preprocess SEC filings uv run nflow run dg_sdg_preprocess --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml -# Stage 1: Generate verified Q&A -uv run nflow run generate_verified_qa --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml +# Stage 1: Generate + verify questions +uv run nflow run generate_verified_questions --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml -# Stage 2: Select best answers -uv run nflow run genselect_answers --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml +# Stage 2: Generate candidate answers +uv run nflow run generate_answers --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml -# Stage 3: Evaluate answers +# Stage 3: Select best answers +uv run nflow run gym_genselect_answers --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml + +# Stage 4: Evaluate answers uv run nflow run evaluate_answers --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml -# Stage 4: Aggregate results +# Stage 5: Aggregate results uv run nflow run aggregate_answers --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml -# Stage 5: Estimate difficulty -uv run nflow run difficulty_estimation --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml - # Stage 6: Post process uv run nflow run dgsdg_post_process --config nvflow/recipes/finance/workflows/sdg/document-grounded-sdg.yaml ``` @@ -121,24 +120,22 @@ ${base_data_dir}/ β”‚ └── jsonl/ β”‚ β”œβ”€β”€ 10-k-data.jsonl # Sampled 10-K data β”‚ └── 10-q-data.jsonl # Sampled 10-Q data -β”œβ”€β”€ step-1-qa-pipeline/ -β”‚ β”œβ”€β”€ question_pipeline/ -β”‚ β”‚ β”œβ”€β”€ generated/ # Generated questions -β”‚ β”‚ └── verified/ # Verified questions -β”‚ └── answer_pipeline/ -β”‚ └── generated/ # Generated answers -β”œβ”€β”€ step-2-genselect/ +β”œβ”€β”€ step-1-questions/ +β”‚ β”œβ”€β”€ generate_input.jsonl # Q-prep output +β”‚ β”œβ”€β”€ generated/ # Generated questions +β”‚ β”œβ”€β”€ verify_input.jsonl # Q-verify-prep output +β”‚ └── verified/ # Verified questions (consumed by step-2) +β”œβ”€β”€ step-2-answers/ +β”‚ β”œβ”€β”€ answer_input.jsonl # A-prep output (threshold-filtered) +β”‚ └── generated/ # Generated answers (consumed by step-3) +β”œβ”€β”€ step-3-genselect/ β”‚ └── selected_answers.jsonl -β”œβ”€β”€ step-3-evaluate/ -β”‚ └── evaluation results (5 seeds) -β”œβ”€β”€ step-4-aggregate/ +β”œβ”€β”€ step-4-evaluate/ +β”‚ └── evaluation results (multi-seed) +β”œβ”€β”€ step-5-aggregate/ β”‚ └── aggregated_answers.jsonl -β”œβ”€β”€ step-5-difficulty/ -β”‚ └── difficulty scoring results └── step-6-post-process/ - β”œβ”€β”€ full_data.jsonl # All cleaned records - β”œβ”€β”€ final_result.jsonl # Medium difficulty (for SFT) - └── hard_rl_data.jsonl # Hard difficulty training data (difficulty_score=0) + └── final_result.jsonl # Cleaned + renamed records consumed by SFT / GRPO ``` ## Expected Results @@ -150,33 +147,32 @@ ${base_data_dir}/ | Questions Generated | ~2M+ | | Verified Questions | ~1.6M | | Final Q&A Pairs | ~800K | -| Medium Difficulty | ~100K | -| Hard Difficulty | ~400K | | Time | ~30 hours, affected by resources used | ## Output Format ### Final Training Data -**final_result.jsonl** - For supervised fine-tuning: -```json -{ - "question": "Based on the risk factors, what are Tesla's main supply chain concerns?", - "context": "...SEC filing excerpt...", - "generation": "...\n...", - "difficulty_score": 2, - "evaluation_score": 4.5 -} -``` +**final_result.jsonl** - Cleaned, renamed records consumed by downstream SFT / GRPO. Each line contains the per-stage allowlisted generic fields (see `nvflow/generic_stage/sdg/document_grounded/_schemas.py::STAGE_KEEP["dgsdg_post_process"]`) plus the recipe-declared `domain_keep_fields`. It also carries the Responses-API *original form* of the selected answer (`response` + `responses_create_params`) and an `expected_answer` mirroring `answer`, so the record is rollout-like and drop-in for SFT / GRPO. Example for the finance recipe: -**hard_rl_data.jsonl** - Hard difficulty training data: ```json { - "question": "How does NVIDIA's revenue recognition differ for bundled products?", - "context": "...complex accounting excerpt...", - "generation": "...\n...", - "difficulty_score": 0, - "evaluation_score": 4.8 + "context": "...SEC filing excerpt...", + "problem": "Based on the risk factors, what are Tesla's main supply chain concerns?", + "answer": "...", + "reasoning_content": "...", + "question_type": "Risk_Factors", + "answerable": "YES", + "question_voting_pass_rate": 1.0, + "question_voting_total": 5, + "expected_answer": "...", + "responses_create_params": { "...": "exact answer-gen request (Responses-API)" }, + "response": { "...": "original answer-gen response object (Responses-API)" }, + "company_name0": "Tesla, Inc.", + "year": "2023", + "item_section0": "Item 1A", + "file_path0": ".../10-K/...", + "file_type": "10-K" } ``` @@ -187,20 +183,14 @@ ${base_data_dir}/ BASE_DIR="outputs/finance/sap-500/workflow-3-document-grounded-sdg" # Stage outputs -ls $BASE_DIR/step-1-qa-pipeline/answer_pipeline/generated/ -ls $BASE_DIR/step-2-genselect/selected_answers.jsonl -ls $BASE_DIR/step-4-aggregate/aggregated_answers.jsonl +ls $BASE_DIR/step-2-answers/generated/ +ls $BASE_DIR/step-3-genselect/selected_answers.jsonl +ls $BASE_DIR/step-5-aggregate/aggregated_answers.jsonl -# Final datasets +# Final dataset ls $BASE_DIR/step-6-post-process/ - -# Count Q&A by difficulty -echo "Medium difficulty:" wc -l $BASE_DIR/step-6-post-process/final_result.jsonl -echo "Hard difficulty:" -wc -l $BASE_DIR/step-6-post-process/hard_rl_data.jsonl - # Inspect samples head -n 3 $BASE_DIR/step-6-post-process/final_result.jsonl | jq . ``` @@ -221,7 +211,8 @@ Converts raw SEC 10-K and 10-Q HTML filings into structured JSONL data for downs |-----------|-------------|---------| | `input_dir` | Raw SEC filings directory (10-K and 10-Q HTML files) | `${filings_dir}/data` | | `output_dir` | Preprocessed data output directory | `${base_data_dir}/step-0-preprocess` | -| `distribution_dir` | Directory with distribution CSVs (SecQue benchmark) | `/workspace/nvflow/recipes/finance/workflows/sdg/dg_sdg_distribution` | +| `distribution_dir` | Directory with distribution CSVs (SecQue benchmark) | `nvflow/recipes/finance/workflows/sdg/dg_sdg_distribution` | +| `preprocess_module` | Dotted module path to domain CLI that chunks + samples | `nvflow.recipes.finance.utils.sdg.dg_sdg_data_preprocess` | | `max_tokens` | Maximum tokens per chunk | 3000 | | `overlap_tokens` | Overlap tokens between chunks for context coverage | 500 | | `total_samples` | Total samples to generate following distribution | 150000 | @@ -251,18 +242,25 @@ ${filings_dir}/data/ This structure is created automatically by the SEC download workflow ([Workflow 1](01-download-sec.md)). -## Stage 1: generate_verified_qa Details +## Stage 1: generate_verified_questions Details -This stage performs 6 internal sub-steps: +This stage performs 4 internal sub-steps (Q-side of the pipeline): -1. **Preprocess Documents** (CPU): Prepare SEC filings for question generation -2. **Generate Questions** (GPU): Create questions from documents using GPT-OSS-120B -3. **Preprocess Questions** (CPU): Prepare for verification -4. **Verify Questions** (GPU): Verify quality using Qwen3-235B (5 seeds) -5. **Preprocess Verified** (CPU): Filter by threshold, prepare for answers -6. **Generate Answers** (GPU): Create answers using GPT-OSS-120B (5 seeds) +1. **Q-prep** (CPU): Run the recipe-supplied `question_prep_script` to attach `context` strings to each chunk +2. **Q-gen** (GPU): Generate questions from documents using GPT-OSS-120B +3. **Q-verify-prep** (CPU): Expand each generated question into N verification trials +4. **Q-verify** (GPU): Per-question Yes/No vote using Qwen3-235B (5 seeds) -See [technical reference](../stages/document-grounded-sdg.md#generate_verified_qa) for details. +See [technical reference](../stages/document-grounded-sdg.md#generate_verified_questions) for details. + +## Stage 2: generate_answers Details + +This stage performs 2 internal sub-steps (A-side of the pipeline): + +1. **A-prep** (CPU): `construct_answer_generate_input` keeps only questions whose Q-verify pass-rate β‰₯ `answer_preprocess_kwargs.threshold` +2. **A-gen** (GPU): Generate N candidate answers per surviving question using GPT-OSS-120B (5 seeds for downstream genselect) + +See [technical reference](../stages/document-grounded-sdg.md#generate_answers) for details. ## Customization @@ -279,11 +277,17 @@ num_chunks: 10 # Change from 1 β†’ 10 to run 10 jobs in parallel ```yaml stages: - generate_verified_qa: + generate_verified_questions: question_generation_kwargs: args: model: /path/to/your/model - server_gpus: 8 + num_gpus: 8 + + generate_answers: + answer_generation_kwargs: + args: + model: /path/to/your/model + num_gpus: 8 ``` ### Modify Prompts @@ -291,9 +295,9 @@ stages: Edit prompts in `nvflow/recipes/finance/prompts/`: - `document_grounded_generate_questions.yaml` - Question generation - `document_grounded_verify_questions.yaml` - Question verification -- `generate_answers.yaml` - Answer generation +- `secque_template.yaml` - Answer generation +- `genselect_answers.yaml` - GenSelect (best-of-N answer picker) - `evaluate_answers.yaml` - Answer evaluation -- `judge_difficulty.yaml` - Difficulty judging ## Common Issues @@ -313,13 +317,6 @@ ls outputs/finance/sap-500/workflow-2-download-sec/step-0-download/data/ - Lower threshold to 0.6 (3 out of 5 seeds) - Review question generation prompt -### Difficulty estimation takes too long - -**Solution:** -- Reduce `num_random_seeds` for answer generation -- Use fewer `num_chunks` for parallelization -- Use smaller judge model - ## Combining with Template-Based You can combine both SDG approaches: @@ -338,7 +335,7 @@ cat outputs/finance/sap-500/workflow-3-template-based-sdg/step-5-filter-answers/ After completing document-grounded SDG: -- **[SFT Training](04-sft.md)** - Train on stratified datasets +- **[SFT Training](04-sft.md)** - Train on `final_result.jsonl` - **[Evaluation](05-eval.md)** - Test model performance - Combine with template-based data for more diversity @@ -353,6 +350,5 @@ For comprehensive stage-by-stage documentation: |-------|-------|------| | GPT-OSS-120B | Question generation, answer generation | 120B | | Qwen3-235B-A22B | Question verification, answer selection, evaluation | 235B | -| Qwen3-4B | Difficulty estimation (small model baseline) | 4B | All models are configurable in the workflow YAML. diff --git a/docs/recipes/finance/workflows/04-sft.md b/docs/recipes/finance/workflows/04-sft.md index 3aa39c4..e871e8d 100644 --- a/docs/recipes/finance/workflows/04-sft.md +++ b/docs/recipes/finance/workflows/04-sft.md @@ -56,19 +56,19 @@ Fine-tune language models on synthetic financial Q&A data generated from SDG wor β”‚ β–Ό β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ 5. convert_to_messages β”‚ Conversion: Convert to OpenAI messages format +β”‚ 5. eval β”‚ Evaluation: Score checkpoints on finance benchmarks β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` -> **Note:** All 6 stages run in the production `qwen3_14b.yaml` configuration. Some stages (`sequence_length_grouping`, `convert_to_messages`) may be optional for custom configurations. - **6 Stages:** 1. **data_transformation** (Step 0): Convert Q&A format to training format 2. **prepare_for_sft** (Step 1): Prepare data for SFT (formatting, filtering) 3. **train_validation_split** (Step 2): Split into train/validation sets 4. **sequence_length_grouping** (Step 3): Group by sequence length for efficiency 5. **training** (Step 4): Fine-tune the model -6. **convert_to_messages** (Step 5): Convert to message format for chat interfaces +6. **eval** (Step 5): Evaluate checkpoints on finance benchmarks + +> **Qwen3 models add a seventh stage.** `qwen3_14b.yaml` inserts `convert_to_messages` between `training` and `eval` to convert checkpoints to the OpenAI messages format. Other configs, including the `qwen3_4b.yaml` demo, run the six stages above. **See [technical reference](../stages/sft.md) for detailed stage documentation.** diff --git a/docs/recipes/finance/workflows/05-eval.md b/docs/recipes/finance/workflows/05-eval.md index 8584e2f..fc7c78c 100644 --- a/docs/recipes/finance/workflows/05-eval.md +++ b/docs/recipes/finance/workflows/05-eval.md @@ -36,6 +36,8 @@ Standalone Baselines └───────────── β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` +> **Offline clusters:** `prepare_data` downloads the benchmark datasets from HuggingFace, so temporarily clear `HF_HUB_OFFLINE`, `HF_DATASETS_OFFLINE` and `TRANSFORMERS_OFFLINE` for its first run, then restore them. The datasets persist and are reused afterwards. See [Offline runtime](../troubleshooting.md#offline-runtime). + ## Configuration **Directory:** `workflows/eval/` @@ -51,7 +53,8 @@ Checkpoint evaluation is configured directly in the training configs: |------|-------------| | `sft/qwen3_4b.yaml` | `stages.eval` with `eval_steps: [10]` | | `sft/qwen3_14b.yaml` | `stages.eval` with `eval_steps: [2600, 5000, 7408]` | -| `grpo/qwen3_4b.yaml` | `stages.eval` with `eval_steps: [20]` | +| `grpo/qwen3_4b.yaml` (equivalence, FSDP) | `stages.eval` with `eval_steps: [20]` | +| `grpo/qwen3_4b_finsec.yaml` (finance_sec_search, Megatron) | `stages.eval` with `eval_steps: [20]` | ## Usage @@ -94,7 +97,8 @@ Configured in `eval/base.yaml`, shared across all evaluation contexts: - **SEC-QUE**: SEC filing comprehension (565 samples) - **FinanceBench**: Financial question answering (150 samples) -- **finance_agent**: Multi-turn agentic financial QA from [vals-ai/finance-agent](https://github.com/vals-ai/finance-agent) (50 samples) + +`finance_agent` (multi-turn agentic financial QA from [vals-ai/finance-agent](https://github.com/vals-ai/finance-agent)) is **disabled** β€” `eval/base.yaml` sets it to `null` pending validation of the multi-turn tool-calling path. See [finance-agent-eval](../stages/finance-agent-eval.md) to re-enable it. ## Eval Stage Configuration (in Training YAMLs) @@ -105,12 +109,12 @@ stages: eval: eval_steps: [1000, 3000, 5000] checkpoint_path: ${directories.step-4-training}/model-name - format: megatron # Use "fsdp" for GRPO demo, "megatron" for GRPO production + format: megatron # Match the checkpoint's training backend: "fsdp" or "megatron" baseline_model: /hf_models/Qwen/Qwen3-14B server_type: vllm gpus: 1 inference_args: >- - ++prompt_config=/workspace/nvflow/recipes/finance/prompts/secque_template.yaml + ++prompt_config=nvflow/recipes/finance/prompts/secque_template.yaml ++inference.temperature=0.6 ++inference.top_p=0.95 ++inference.top_k=20 @@ -170,12 +174,12 @@ stages: eval: eval_steps: [100, 500, 1000] checkpoint_path: ${directories.step-4-training}/model-my-model-name - format: megatron # Use "fsdp" for GRPO demo checkpoints + format: megatron # Match the checkpoint's training backend: "fsdp" or "megatron" baseline_model: /hf_models/MyOrg/MyModel server_type: vllm gpus: 1 inference_args: >- - ++prompt_config=/workspace/nvflow/recipes/finance/prompts/secque_template.yaml + ++prompt_config=nvflow/recipes/finance/prompts/secque_template.yaml server_args: "--max-model-len 40960" ``` diff --git a/docs/recipes/finance/workflows/06-finance-agent-eval.md b/docs/recipes/finance/workflows/06-finance-agent-eval.md deleted file mode 100644 index 2d975c7..0000000 --- a/docs/recipes/finance/workflows/06-finance-agent-eval.md +++ /dev/null @@ -1,64 +0,0 @@ -# Finance Agent Benchmark - -## Overview - -The **finance_agent** benchmark ([vals-ai/finance-agent](https://github.com/vals-ai/finance-agent)) is now integrated into the main [evaluation workflow](05-eval.md). It is defined as a benchmark entry in `workflows/eval/base.yaml` alongside SEC-QUE and FinanceBench. - -> **Note:** The standalone `finance_agent_eval.yaml` workflow has been removed. All finance_agent evaluation now runs through the unified eval configs in `workflows/eval/`. - -## What is finance_agent? - -- **50 public questions** from vals-ai/finance-agent -- **Multi-turn**: Model can take up to 50 turns (tool calls + reasoning) -- **Tools**: Web search (Tavily), SEC EDGAR lookup, HTML parsing -- **Judge**: GPT-5 mini with strict finance-domain prompts (`sec_judge_strict.yaml`) - -## Configuration - -The finance_agent benchmark is configured in `workflows/eval/base.yaml` under the `benchmarks` section: - -```yaml -benchmarks: - finance_agent: - seeds: 5 - judge: *judge_finance_strict - installation_command: "pip install -q ..." - extra_args: >- - ++max_turns=50 - ++inference.tokens_to_generate=32000 - ++inference.temperature=0.0 - ++max_concurrent_requests=1 -``` - -Any model YAML that inherits from `base.yaml` will automatically include finance_agent in its evaluation benchmarks. - -## Usage - -Run finance_agent evaluation as part of any eval context: - -```bash -# Evaluate baselines on all benchmarks (including finance_agent) -uv run nflow run-all --config nvflow/recipes/finance/workflows/eval/baselines.yaml - -# SFT training + checkpoint eval (includes finance_agent) -uv run nflow run-all --config nvflow/recipes/finance/workflows/sft/qwen3_14b.yaml -``` - -## Output Structure - -Outputs appear under the model's eval-results directory: - -``` -outputs/finance/sap-500/workflow-1-baseline-eval/ -└── baselines/ - └── gpt-oss-120b/ - └── eval-results/ - └── finance_agent/ - β”œβ”€β”€ metrics.json # Aggregated metrics - └── output*.jsonl # Predictions per seed -``` - -## Related - -- **[Evaluation Workflow (05-eval)](05-eval.md)** – Full eval documentation, including all benchmarks -- **[Eval Stages Reference](../stages/eval.md)** – Technical stage documentation diff --git a/docs/recipes/finance/workflows/06-grpo.md b/docs/recipes/finance/workflows/06-grpo.md index 7e263d9..a04d2a0 100644 --- a/docs/recipes/finance/workflows/06-grpo.md +++ b/docs/recipes/finance/workflows/06-grpo.md @@ -96,10 +96,11 @@ Further improve fine-tuned models using Group Relative Policy Optimization (GRPO ### Model Configurations -| Config | Model | GPUs | Status | -|--------|-------|------|--------| -| `grpo/qwen3_4b.yaml` | Qwen3-4B | 16 (2 nodes) | Demo | -| `grpo/qwen3_30b_a3b.yaml` | Qwen3-30B-A3B (MoE) | 64 (8 nodes) | Production | +| Config | Model | Environment | Backend | GPUs | Status | +|--------|-------|-------------|---------|------|--------| +| `grpo/qwen3_4b.yaml` | Qwen3-4B | equivalence_llm_judge | FSDP v2 (32K) | 16 (2 nodes) | Demo | +| `grpo/qwen3_4b_finsec.yaml` | Qwen3-4B | finance_sec_search | Megatron (TP2Γ—CP8, 131K) | 16 (2 nodes) | Demo | +| `grpo/qwen3_30b_a3b.yaml` | Qwen3-30B-A3B (MoE) | β€” | Megatron | 64 (8 nodes) | Production | ## Usage @@ -220,9 +221,12 @@ outputs/finance/demo/workflow-5-grpo/ β”‚ β”œβ”€β”€ val.jsonl # Validation split β”‚ └── logs/ β”œβ”€β”€ step-8-training/ - β”‚ └── grpo-qwen3-4b-2n-tp2-cp4-seq131k/ # Demo (FSDP v2) - β”‚ β”œβ”€β”€ checkpoints/ # GRPO model checkpoints - β”‚ └── training-logs/ + β”‚ β”œβ”€β”€ equivalence_llm_judge/ + β”‚ β”‚ └── grpo-qwen3-4b-16g-tp2-cp1-seq32k/ # Demo, FSDP v2 + β”‚ └── finance_sec_search/ + β”‚ └── grpo-qwen3-4b-16g-tp2-cp8-seq128k/ # Demo, Megatron (YaRN 131K) + β”‚ β”œβ”€β”€ checkpoints/ # GRPO model checkpoints + β”‚ └── training-logs/ └── step-9-eval/ └── ... # Benchmark evaluation results ``` @@ -341,7 +345,7 @@ stages: ### Training Backends -The demo config (`qwen3_4b.yaml`) uses **FSDP v2** for the dense Qwen3-4B model. The production config (`qwen3_30b_a3b.yaml`) uses **Megatron** for the Qwen3-30B-A3B MoE model at 64 GPUs. +The demo runs two environments with different backends: `qwen3_4b.yaml` (equivalence_llm_judge) uses **FSDP v2** at 32K, while `qwen3_4b_finsec.yaml` (finance_sec_search) uses **Megatron** (TP2Γ—CP8) for YaRN context extension to 131K. The production config (`qwen3_30b_a3b.yaml`) uses **Megatron** for the Qwen3-30B-A3B MoE model at 64 GPUs. **Production (Megatron):** diff --git a/docs/recipes/multimodal/README.md b/docs/recipes/multimodal/README.md new file mode 100644 index 0000000..2e279f7 --- /dev/null +++ b/docs/recipes/multimodal/README.md @@ -0,0 +1,112 @@ +# Multimodal HopChain Recipe + +The multimodal recipe implements a HopChain-inspired synthetic data generation +pipeline for multi-hop vision-language reasoning. It follows the paper +[HopChain: Multi-Hop Data Synthesis for Generalizable Vision-Language Reasoning](https://arxiv.org/pdf/2603.17024) +and expresses the workflow as reusable NVFlow stages. + +Start with the [HopChain quick start](quick-start.md). + +## Workflows + +| Workflow | Demo config | Full config | +| --- | --- | --- | +| Image filter | `nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml` | `nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml` | +| SDG | `nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml` | `nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml` | + +The demo SDG config stops after verified-question visualization. It has no +external API dependency. The full config additionally runs the OpenAI +judge, reconciliation, Omni difficulty filtering, and SFT trace generation. + +## Configuration Contract + +Configuration is split between workflow and cluster files: + +- Each full workflow YAML defines its stages, model profiles, execution + IDs, chunking, and repository-relative input/output paths. +- Each demo YAML inherits its corresponding full workflow and overrides + only the stage selection and small-run settings. +- [`cluster_configs/my_cluster.yaml`](../../cluster-configuration.md) defines + the local Slurm account, partitions, mounts, and named container image paths. +- Optional private recipe changes go in git-ignored `private_*.yaml` overlays + next to the workflow they modify. +- The full workflow's OpenAI key is supplied as `OPENAI_API_KEY` under `env_vars` + in `cluster_configs/my_cluster.yaml`. + +Run the demo workflows in order: + +```bash +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml + +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml +``` + +Outputs are deterministic: + +```text +outputs/hopchain/image_filter/execution/demo/ +outputs/hopchain/sdg/execution/demo/ +``` + +Run the full workflows with their full configs: + +```bash +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml + +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml +``` + +Full-workflow outputs use these directories: + +```text +outputs/hopchain/image_filter/execution/full/ +outputs/hopchain/sdg/execution/full/ +``` + +## SDG Stages + +The full `hopchain_sdg` workflow runs: + +1. `prepare_filtered_image_inputs` +2. `preprocess_identify_categories` +3. `identify_categories` +4. `localize_instances` +5. `sample_instance_combinations` +6. `preprocess_generate_multihop_queries` +7. `generate_multihop_queries` +8. `verify_candidate_queries` +9. `visualize_candidate_hopchain_data` +10. `judge_candidate_queries_openai` +11. `reconcile_llm_judges` +12. `visualize_reconciled_hopchain_data` +13. `preprocess_filter_easy_candidates` +14. `filter_easy_candidates` +15. `preprocess_generate_sft_reasoning_traces` +16. `generate_sft_reasoning_traces` +17. `preprocess_filter_sft_reasoning_traces` +18. `filter_sft_reasoning_traces` + +## Inputs and Models + +The image filter recursively scans `data/images/` and writes +`outputs/hopchain/image_filter/execution/demo/image-filter/kept_images.jsonl`. +The SDG demo reads that file as its input. + +By default the containers must see checkpoints at: + +```text +/hf_models/Qwen/Qwen3.5-397B-A17B +/hf_models/facebook/sam3.1/sam3.1_multiplex.pt +/hf_models/nvidia/omni-step70 +``` + +Set host-to-container mappings in `my_cluster.yaml` and server behavior in an +ignored local YAML overlay. Keep machine-specific paths in those local files. + +Do not commit API keys or credential files. See the +[quick start](quick-start.md#run-the-full-workflow) for full-workflow credential setup +and the data-egress warning. diff --git a/docs/recipes/multimodal/quick-start.md b/docs/recipes/multimodal/quick-start.md new file mode 100644 index 0000000..b17399b --- /dev/null +++ b/docs/recipes/multimodal/quick-start.md @@ -0,0 +1,326 @@ +# HopChain Quick Start + +Run HopChain from a folder of images to verified multi-hop vision-language +questions. + +The demo has two commands: + +1. Filter the source images with Qwen. +2. Generate and verify multi-hop questions with Qwen and SAM 3.1. + +Plan for 30–60 minutes for a small demo run, plus Slurm queue time. Image +filtering typically takes 5–10 minutes, and the SDG dependency chain takes 20 +minutes or more. Runtime increases with the number of images and generated +queries. + +The demo ends after verified-question visualization. The full workflow +adds the OpenAI judge and Omni curation stages. + +## Pipeline Overview + +```text + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ 1. Your images │────▢│ 2. Image Filter │────▢│ 3. SDG β”‚ + β”‚ (a folder) β”‚ β”‚ keep complex β”‚ β”‚ categories β†’ localize β†’ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ combine β†’ generate & β”‚ + β”‚ β”‚ verify multi-hop queries β”‚ + kept_images.jsonl β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ 4. Judge + reconcile β”‚ β”‚ 5. Difficulty filter β”‚ + β”‚ (OpenAI API) β”‚ β”‚ + SFT reasoning tracesβ”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +The full path continues from verified questions through the external +judge, judge reconciliation, difficulty filtering, and SFT reasoning-trace +generation. Image filtering is a separate workflow so its output can be reused +by several SDG runs. + +## Prerequisites + +- Cluster access configured as described in [INSTALL.md](../../../INSTALL.md), + with all commands run from the `nvflow` repository root. +- A Slurm cluster config created from `cluster_configs/template-slurm.yaml`; see + [Step 1](#1-configure-models-and-cluster). +- GPUs for the model workers. The **core** path (Steps 3–4) needs: + - A **VLM server** for image scoring and question generation. The reference + config serves `Qwen/Qwen3.5-397B-A17B` with SGLang + - A **SAM 3.1 worker** for object localization. +- The **full** path additionally needs an OpenAI API key for the LLM judge and + the Omni reasoning VLM for difficulty filtering. + +> **Heads up:** The reference models are large. For a quick try, use +> [smaller models you can serve](#local-overrides). + +## 1. Configure Models and Cluster + +### SAM 3.1 checkpoint + +Request access to +[Meta's gated SAM 3.1 repository](https://huggingface.co/facebook/sam3.1), then +download the checkpoint once from a connected host: + +```bash +uv run hf auth login +uv run hf download facebook/sam3.1 sam3.1_multiplex.pt \ + --local-dir /path/to/models/hf_models/facebook/sam3.1 +``` + +After the checkpoint is downloaded, the compute jobs do not need `HF_TOKEN`. + +### Cluster configuration + +Follow [Configure Your Cluster](../../../INSTALL.md#configure-your-cluster) to +create `cluster_configs/my_cluster.yaml`. The +[Cluster Configuration Guide](../../cluster-configuration.md) documents every +available field. + +In `my_cluster.yaml`, configure the named `nemo-skills`, `sglang`, and `vllm` +container entries. Mount the checkout and your host model directory so they +are visible on every compute node. The reference configs expect these paths +inside the containers: + +```text +/hf_models/Qwen/Qwen3.5-397B-A17B +/hf_models/facebook/sam3.1/sam3.1_multiplex.pt +/hf_models/nvidia/omni-step70 # full workflow only +``` + +The workflow resolves the repository root from the shell's standard `PWD`. +Make the checkout visible to Slurm jobs at the same absolute path. On sites +that mount a workspace at `/workspace`, launch NVFlow from the checkout under +that mount, such as `/workspace/nvflow`. + +## 2. Add Images + +Copy or mount images anywhere below: + +```bash +mkdir -p data/images +# Copy or mount images below data/images/. +``` + +Subdirectories are scanned recursively. The demo selects at most 100 images and +the SDG step uses at most 25 images that pass filtering. Prefer visually rich +scenes, documents, charts, or infographics with several distinct regions. + +Public datasets that fit the recipe well include: + +| Dataset | Why it fits HopChain | Source | +| --- | --- | --- | +| COCO 2017 validation | Everyday multi-object scenes; a practical first run | | +| Visual Genome | Dense objects and relationships | | +| InfographicVQA, DocVQA, or ChartQA | Text- and figure-rich images for OCR reasoning | Hugging Face Datasets | +| ADE20K | Complex scene-parsing images | | +| Open Images V7 | Large and diverse multi-object collection | | + +Images remain path-referenced throughout the pipeline, so keep the directory +mounted and unchanged until the run completes. + +## 3. Filter Images (~5–10 minutes) + +Validate, preview, and submit the demo: + +```bash +uv run nflow validate \ + --config nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml + +uv run nflow list-stages \ + --config nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml + +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml +``` + +`run-all` submits Slurm work and returns. After the job finishes, inspect the +deterministic demo output: + +```bash +python -m json.tool outputs/hopchain/image_filter/execution/demo/image-filter/summary.json +wc -l outputs/hopchain/image_filter/execution/demo/image-filter/kept_images.jsonl +``` + +The second command must report at least one kept image before SDG can proceed. + +The image-filter output contains: + +```text +image-filter/ +β”œβ”€β”€ image_catalog.jsonl +β”œβ”€β”€ output.jsonl +β”œβ”€β”€ final_output.jsonl +β”œβ”€β”€ kept_images.jsonl +└── summary.json +``` + +`final_output.jsonl` includes every scored image; `kept_images.jsonl` contains +only images that passed the configured quality and complexity thresholds. + +## 4. Generate Multi-Hop Questions (~20+ minutes) + +The SDG demo reads +`outputs/hopchain/image_filter/execution/demo/image-filter/kept_images.jsonl`. + +```bash +uv run nflow validate \ + --config nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml + +uv run nflow list-stages \ + --config nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml + +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml +``` + +The demo runs the local core path: + +```text +prepare images -> identify categories -> localize with SAM -> sample object +combinations -> generate questions -> verify questions -> build visualization +``` + +After the dependency chain completes: + +```bash +python -m json.tool \ + outputs/hopchain/sdg/execution/demo/step-5-verify-candidate-queries/summary.json + +wc -l \ + outputs/hopchain/sdg/execution/demo/step-5-verify-candidate-queries/final_candidates.jsonl +``` + +Review the generated HTML under +`outputs/hopchain/sdg/execution/demo/step-6-visualize-candidate-hopchain-data/`. + +The core output layout is: + +```text +sdg/execution/demo/ +β”œβ”€β”€ step-0-prepare-filtered-inputs/filtered_image_inputs.jsonl +β”œβ”€β”€ step-1-identify-categories/final_output.jsonl +β”œβ”€β”€ step-2-localize-instances/ +β”œβ”€β”€ step-3-sample-instance-combinations/instance_combinations.jsonl +β”œβ”€β”€ step-4-generate-multihop-queries/final_output.jsonl +β”œβ”€β”€ step-5-verify-candidate-queries/ +β”‚ β”œβ”€β”€ final_candidates.jsonl +β”‚ β”œβ”€β”€ rejected_candidates.jsonl +β”‚ └── summary.json +└── step-6-visualize-candidate-hopchain-data/ +``` + +## Run the Full Workflow + +The full configs use the `full` execution ID. They process the complete input +set, use full-run chunk counts, call the +OpenAI judge, run the Omni +difficulty filter, and create SFT reasoning traces. + +Before running the full workflow, add your OpenAI key to +`cluster_configs/my_cluster.yaml`, following the existing +[environment-variable instructions](../../cluster-configuration.md#environment-variables): + +```yaml +env_vars: + # ...existing cluster environment variables... + - OPENAI_API_KEY= +``` + +The OpenAI judge sends question and image content to an external service. Only +enable the full path when that data transfer is allowed. + +Then run: + +```bash +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml + +uv run nflow run-all \ + --config nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml +``` + +Full-workflow outputs live under: + +```text +outputs/hopchain/image_filter/execution/full/ +outputs/hopchain/sdg/execution/full/ +``` + +The full-workflow stage groups are: + +| Steps | Work | Needs | +| --- | --- | --- | +| 0–6 | Prepare, identify, localize, combine, generate, verify, visualize | Qwen and SAM | +| 7–9 | OpenAI judge, reconcile, and visualize reconciled data | `OPENAI_API_KEY` | +| 10 | Filter easy candidates | Omni reasoning VLM | +| 11–12 | Generate and filter SFT reasoning traces | Qwen | + +Adjust full-run chunk counts after checking the image-filter and combination +counts for your dataset. + +## Local Overrides + +Put deployment-specific recipe changes in a small `private_*.yaml` overlay next +to the workflow it modifies (`private_*.yaml` files are git-ignored repo-wide). +For example: + +```yaml +# nvflow/recipes/multimodal/workflows/image_filter/private_hopchain-image-filter.yaml +_base_: hopchain-image-filter-demo.yaml + +execution_id: my_test +model_profiles: + qwen: + server_gpus: 4 + server_nodes: 1 + server_args: >- + --model-path /hf_models/Qwen/Qwen3.5-397B-A17B + --served-model-name qwen3.5-397b-a17b + --tp 4 + --trust-remote-code +``` + +Use another small overlay based on `hopchain-sdg-demo.yaml` (in +`workflows/sdg/`) when the SDG model profile also needs to change. Keep host +paths, Slurm partitions, mounts, and container image paths in +`cluster_configs/my_cluster.yaml`. + +## Next Steps + +- Review `final_candidates.jsonl` and the candidate HTML before enabling the + external judge. +- Tune `min_complexity_score` or `allowed_quality_ratings` in a local + image-filter overlay when the kept set is too broad or too small. +- Use a local SDG overlay to calibrate `sample_count`, query count, and chunk + counts before a full run. +- Read the [multimodal HopChain guide](README.md) for the complete stage list + and configuration contract. + +## Troubleshooting + +### The config validates, but the job cannot see files + +`validate` runs in the launch shell; the stage itself runs in a container on a +compute node. Confirm that the checkout, images, outputs, and checkpoint paths +are covered by `my_cluster.yaml` mounts and appear at the paths documented +above. + +### No images were selected + +Confirm `data/images/` contains supported image files. If filtering ran +but kept zero images, inspect `final_output.jsonl` and lower +`min_complexity_score` in a local image-filter overlay. + +### A job requests the wrong partition or container + +Partitions and container image paths come from `cluster_configs/my_cluster.yaml`. +Check `partition`, `cpu_partition`, and the named container entries there. + +### The full workflow fails at the judge stage + +Confirm `OPENAI_API_KEY` is present under `env_vars` in the ignored +`cluster_configs/my_cluster.yaml`. The cluster config injects it into the +`nemo-skills` container used by the full-workflow judge. + +[Multimodal HopChain Guide](README.md) | [Main README](../../../README.md) diff --git a/docs/remote-launch.md b/docs/remote-launch.md new file mode 100644 index 0000000..b1712db --- /dev/null +++ b/docs/remote-launch.md @@ -0,0 +1,152 @@ +# Running `nflow` over an SSH tunnel + +`nflow` is only a **submission orchestrator**: it builds Slurm jobs and submits +them β€” all data, GPU work, and training run in the worker containers **on the +cluster**. When you run `nflow` somewhere that can't reach Slurm directly (a +laptop, a dev box, or an isolated/airgapped environment), it submits over an +**SSH tunnel**. + +> **On a cluster login/dev node?** You don't need this doc β€” install per the +> [README](../README.md#-installation) and run `nflow` directly. This page is for +> the **off-cluster / tunneled** case. For all client options at a glance, see +> [INSTALL.md β†’ Choose your client setup](../INSTALL.md#choose-your-client-setup). + +## Options at a glance + +```text +Where does `nflow` run? +β”‚ +β”œβ”€ On a cluster login/dev node ────────────▢ sbatch ─▢ Slurm worker jobs (no tunnel) +β”‚ install: uv sync +β”‚ +└─ Off-cluster (laptop / dev box / airgap) ──ssh_tunnel──▢ login node ─sbatch─▢ workers + provision the launcher, pick one: + A. host install β€” uv sync (client host needs internet) + B. nvflow-client image β€” no uv sync, no client internet + β”œβ”€ enroot (cluster node) + β”œβ”€ docker/podman (off-cluster machine) + └─ pyxis srun (cluster node, via Slurm) + +Worker jobs (nemo-skills Β· vllm Β· vllm-grpo Β· nemo-rl Β· nemo-gym Β· sglang) +always run on the cluster; the client only submits. +``` + +## Prerequisites + +- **Cluster side is set up** ([INSTALL.md](../INSTALL.md)): worker `.sqsh` images + and models are staged, and you have a `my_cluster.yaml`. +- **SSH key auth** to a cluster login node that can run `sbatch`: + ```bash + ssh -i @ 'hostname && command -v sbatch' + ``` + +## Step 1 β€” Configure `my_cluster.yaml` (add the tunnel) + +Put `my_cluster.yaml` where the launcher reads it β€” **container:** the mounted +`/work` dir (`NEMO_SKILLS_CONFIG_DIR=/work`); **host install:** `cluster_configs/`. +Add an `ssh_tunnel` block so the launcher reaches Slurm over SSH (no Slurm client +or Lustre needed on the client): + +```yaml +ssh_tunnel: + host: + user: + identity: # container: /opt/ssh/ (id_rsa / id_ed25519) + job_dir: +``` + +> `/work` is a **bind mount** β€” prepare `my_cluster.yaml` before starting the +> container, or edit it live afterward; it just must be complete before +> `nflow run`. It holds secrets: keep it in `/work`, never bake it into an image. + +The rest of `my_cluster.yaml` is your standard cluster config (containers, +`mounts:`, `env_vars`); `ssh_tunnel` is the only tunnel-specific addition. In +`mounts:`, keep `/hf_models` and point `/workspace` at a **writable data dir** +(outputs + HF cache) β€” **not** the repo checkout. Recipe code and checked-in +assets reach workers via the packaged snapshot (`/nemo_run/code`), so the repo is +never mounted. See [cluster-configuration.md β†’ Mounts](cluster-configuration.md#mounts). + +## Step 2 β€” Start the launcher (pick one) + +### A. Host install (`uv sync`) β€” client host has internet + +Follow the [README install](../README.md#-installation) (`git clone` + `uv sync`). +Invoke the CLI as **`uv run nflow …`**. No client internet? Use the +`nvflow-client` image (option B below) instead. + +### B. Client container β€” airgapped / no local install (invoke as **`nflow …`**) + +The `nvflow-client` image bundles the `nflow` CLI + venv (no `uv sync`, no client +internet). Start it, mounting your **SSH key** (`β†’ /opt/ssh`) and the **`/work`** +dir holding `my_cluster.yaml`: + +```bash +# --- Cluster node (enroot) β€” if the .sqsh is already staged, skip the import --- +enroot import -o nvflow-client.sqsh 'docker://#/nvflow-client:' # only from a registry ref +enroot create --name nvflow-client /path/to/nvflow-client.sqsh +ENROOT_MOUNT_HOME=n enroot start --rw \ + -m ~/.ssh:/opt/ssh -m /path/to/work:/work \ + -e NEMO_SKILLS_CONFIG_DIR=/work nvflow-client bash + +# --- Cluster node via Slurm (pyxis/srun) β€” starts from the .sqsh directly --- +srun --container-image=/path/to/nvflow-client.sqsh \ + --container-mounts=/path/to/work:/work,$HOME/.ssh:/opt/ssh \ + --container-workdir=/opt/nvflow \ + --export=ALL,NEMO_SKILLS_CONFIG_DIR=/work --pty bash + +# --- Off-cluster machine (docker/podman) --- +docker run --rm -it -v ~/.ssh:/opt/ssh:ro -v /path/to/work:/work \ + -e NEMO_SKILLS_CONFIG_DIR=/work /nvflow-client: bash +``` + +> Prefer **enroot** (cluster) or **docker/podman** (off-cluster); the `srun` form +> burns an allocation just to host the launcher. Do **not** bind-mount over +> `/opt/nvflow` (baked source/venv/`.git` that nemo-run packages via `git archive`). +> Host keys auto-accept on first connect (baked `ssh_config` reads +> `/opt/ssh/known_hosts`; a *changed* key is still rejected). Build details: +> [containers.md](maintainers/containers.md). + +## Step 3 β€” Launch and monitor over the tunnel + +```bash +nflow list-stages --recipe finance # verify: CLI loads + config resolves +nflow run -c -e # submit (detaches when queued) +``` + +The client has **no Slurm client or cluster filesystem**, so monitor on the +cluster over the same SSH: + +```bash +ssh -i @ 'squeue --me' # or: sacct -j +ssh -i @ 'ls /...' # logs/artifacts land on Lustre +``` + +`nemo experiment status ` (printed at submit) also works over the tunnel. + +## Notes + +- **Connected-node prerequisites** (benchmark datasets, SEC filings, model + downloads) need internet and the `HF_*_OFFLINE` flags **off** for that one run β€” + do them once per [INSTALL.md](../INSTALL.md), then keep the flags **on**. The + container can stage models itself: + `uv run hf download --local-dir /hf_models/` (mount the models dir). +- **Everything runs on the cluster; the client only submits.** GPU work, data + I/O, and the rollout/judge servers all execute inside Slurm jobs. Recipe code + and checked-in assets ship with each job via `/nemo_run/code` (see Step 1), so + the client needs no repo and the repo is never mounted on workers. +- **Laptop / off-cluster specifics** (validated: a client with **no repo mount** + ran the full matrix end-to-end β€” staging β†’ SDG β†’ SFT β†’ eval and **both GRPO + workflows** (`finance_sec_search` via the client, equivalence via a repo + install) β€” proving all I/O is cluster-side and checked-in assets resolve from + `/nemo_run/code`, incl. Gym `config_paths`, prefetch `ticker`, and judge + fpaths): + - `ssh_tunnel.host` must be an **FQDN reachable from the laptop** (VPN), and + `ssh_tunnel.identity` your **local** key (e.g. `~/.ssh/id_rsa`). + - `mounts:` and `job_dir` are **cluster Lustre paths**; the laptop needs none of + them locally. Resume/chunk-skip is probed over the tunnel (`LauncherFS`), so + **no local mount is required** β€” and while `ssh_tunnel` is set a local mount + is ignored anyway. (A client running **on-cluster without** `ssh_tunnel` must + run from the repo root so `resolve_host_path` can map `/workspace/outputs/...` + back to the host outputs dir for skip-detection.) + - Dev-mode source overlays (Gym / NeMo-RL) must live **on the cluster**, not the + laptop β€” they bind into the worker jobs. diff --git a/docs/trace-viewer.md b/docs/trace-viewer.md new file mode 100644 index 0000000..67bee64 --- /dev/null +++ b/docs/trace-viewer.md @@ -0,0 +1,61 @@ +# Rollout Trace Viewer + +A lightweight, dependency-free web UI to spot-check NeMo-Gym rollout traces one +record at a time. Implemented in [`scripts/view_traces.py`](../scripts/view_traces.py) +(pure Python stdlib -- no Gradio, no extra installs). + +It reads only the requested record (seek-by-line with a lazy byte-offset cache), +so it opens record 0 or record 35,000 of a multi-GB `output-rs*.jsonl` without +loading the file. + +## Run + +```bash +cd nvflow +uv run python scripts/view_traces.py [--root ] [--port 8800] +``` + +- `--root` (default: `$NVFLOW_TRACE_ROOT` if set, else the current directory) -- + directory scanned for `*.jsonl` files (the file dropdown). Heavy/non-trace dirs + (`cache/`, `logs/`, `.venv/`, ...) and input artifacts + (`*materialized_inputs*`, `*chunk_input*`) are skipped automatically. + Point it at a single workflow output dir. Do **not** point it at a parent that + also holds the SEC filing dump -- scanning tens of thousands of filings makes + the directory listing crawl. +- `--port` (default 8800), `--host` (default `127.0.0.1`). + +## View it in the browser + +The server binds `127.0.0.1`, so reach it through the SSH tunnel: + +- In **Cursor / VS Code Remote**: the port is auto-forwarded. Open the **Ports** + panel, find the port, click the globe ("Open in Browser"). If it isn't listed, + "Forward a Port" -> enter the port. (Start the server in Cursor's integrated + terminal so auto-forward triggers.) +- Manual fallback from your laptop: `ssh -L 8800:localhost:8800 ` then open + `http://localhost:8800`. + +## Using it + +- **File dropdown**: pick a rollout file. For traces choose + `…/rollout/output-rs*.jsonl` or the curated `…/rollout/analysis_rs*/{best,worst,intermediate}.jsonl`. + A `train.jsonl` has no trace (just question + difficulty) and renders as a + collapsible JSON record. +- **Navigate one record at a time**: record-number box + **Go**, **Prev/Next**, + **Random** (Random counts the file once, then is instant). +- **Trace rendering**: the exact recorded order of `input` + `response.output` -- + each step color-coded with an icon/pill (user, reasoning, tool call, tool + output, assistant), collapsed by default with a one-line preview. Click a step + to expand; **Expand all / Collapse all** at the top right. +- **JSON as a tree**: tool-call args, tool outputs, and the **Raw JSON** view + render as a colorized, collapsible tree -- click any `{}`/`[]` to fold/unfold + nested fields. +- **Verdict header**: reward badge, judge rating/text, expected answer, + question type, uuid. + +## Notes + +- Stdlib only; runs under `uv run python` (3.12) or any `python3` (3.9+). +- Responses use `Cache-Control: no-store`, so a plain refresh always shows the + latest after a server restart (restart the server to pick up code edits). +- Single-user local tool: it serves on localhost only and reads files read-only. diff --git a/nvflow/core/__init__.py b/nvflow/core/__init__.py index a3e88f8..20059e5 100644 --- a/nvflow/core/__init__.py +++ b/nvflow/core/__init__.py @@ -14,12 +14,30 @@ # """Core infrastructure for workflow orchestration.""" +from typing import TYPE_CHECKING, Any + from nvflow.core import console from nvflow.core.base_stage import BaseStage from nvflow.core.stage_registry import StageRegistry -from nvflow.core.workflow_runner import WorkflowRunner + +if TYPE_CHECKING: + from nvflow.core.workflow_runner import WorkflowRunner __all__ = ["BaseStage", "StageRegistry", "WorkflowRunner", "console"] + +def __getattr__(name: str) -> Any: + # WorkflowRunner pulls in omegaconf, which is absent from minimal worker + # containers (e.g. the SAM localization image). Those workers import only + # leaf helper modules under nvflow.recipes, and recipe auto-discovery + # touches this package -- so importing WorkflowRunner eagerly here would + # crash them with ModuleNotFoundError. Resolve it lazily instead. + if name == "WorkflowRunner": + from nvflow.core.workflow_runner import WorkflowRunner + + return WorkflowRunner + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + # Note: nemo-skills functions are imported directly in stage files when needed: # from nemo_skills.pipeline.cli import generate, run_cmd, wrap_arguments diff --git a/nvflow/core/workflow_runner.py b/nvflow/core/workflow_runner.py index cc6843a..c425f50 100644 --- a/nvflow/core/workflow_runner.py +++ b/nvflow/core/workflow_runner.py @@ -275,8 +275,9 @@ def run( self._run_stage(stage_name, environment=environment, stages_to_run=stages_to_run) completed_stages.append(stage_name) - header("βœ… Workflow Complete!") - success(f"Completed {len(completed_stages)} stage(s): {', '.join(completed_stages)}") + header("βœ… Workflow Submitted") + success(f"Submitted {len(completed_stages)} stage(s): {', '.join(completed_stages)}") + detail("Note", "Stages run as Slurm jobs -- track them with squeue") def _preflight_pipeline_health( self, diff --git a/nvflow/generic_stage/__init__.py b/nvflow/generic_stage/__init__.py new file mode 100644 index 0000000..efbef28 --- /dev/null +++ b/nvflow/generic_stage/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Shared stage implementations reusable across recipes.""" diff --git a/nvflow/generic_stage/sdg/__init__.py b/nvflow/generic_stage/sdg/__init__.py new file mode 100644 index 0000000..5e74ab1 --- /dev/null +++ b/nvflow/generic_stage/sdg/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Shared SDG stage implementations.""" diff --git a/nvflow/generic_stage/sdg/document_grounded/__init__.py b/nvflow/generic_stage/sdg/document_grounded/__init__.py new file mode 100644 index 0000000..0ac2c98 --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/__init__.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Shared DG-SDG stages and per-recipe registration helper.""" + +from nvflow.core import StageRegistry + +from .aggregate_answers import AggregateAnswersStage +from .dg_sdg_preprocess import DGSDGPreprocessStage +from .dgsdg_post_process import DGSDGPostProcessStage +from .evaluate_answers import EvaluateAnswersStage +from .generate_answers import GenerateAnswersStage +from .generate_verified_questions import GenerateVerifiedQuestionsStage +from .gym_genselect_answers import GymGenselectAnswersStage + +WORKFLOW = "document_grounded_sdg" +SHARED_STAGES: list[tuple[type, str]] = [ + (AggregateAnswersStage, "aggregate_answers"), + (EvaluateAnswersStage, "evaluate_answers"), + (GymGenselectAnswersStage, "gym_genselect_answers"), + (GenerateVerifiedQuestionsStage, "generate_verified_questions"), + (GenerateAnswersStage, "generate_answers"), + (DGSDGPreprocessStage, "dg_sdg_preprocess"), + (DGSDGPostProcessStage, "dgsdg_post_process"), +] + + +def register_for_recipe(recipe: str) -> None: + """Register all shared DG-SDG stages for a concrete recipe name.""" + for stage_class, stage_name in SHARED_STAGES: + if StageRegistry.has(recipe=recipe, workflow=WORKFLOW, stage=stage_name): + raise ValueError( + f"register_for_recipe({recipe!r}) would re-register " + f"{recipe}.{WORKFLOW}.{stage_name}. " + "This usually means old per-recipe shim modules are still imported " + "or the helper was called twice." + ) + StageRegistry.register(recipe=recipe, workflow=WORKFLOW, stage=stage_name)(stage_class) diff --git a/nvflow/generic_stage/sdg/document_grounded/_helpers.py b/nvflow/generic_stage/sdg/document_grounded/_helpers.py new file mode 100644 index 0000000..47e6565 --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/_helpers.py @@ -0,0 +1,365 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Shared helpers for generic DG-SDG stages.""" + +import json +import shlex +from typing import Any + +from ._schemas import ALWAYS_DROP, STAGE_KEEP + + +def clean_stale_experiments(cluster: str, expnames: list[str]) -> None: + """Remove ``/experiments//`` for each name in *expnames*. + + ``rollout()`` (reused unmodified from RL) does not clean stale nemo-run + experiment dirs, but SDG needs it: nemo-run caches the generated bash + scripts per experiment, so a stale dir makes (a) code edits silently + no-op (cached scripts re-used; SKILL.md Gotcha #8) and (b) ``run_after`` + resolve to a stale FINISHED experiment, skipping the Slurm dependency + (Gotcha #1). Replicated here on the SDG side so RL code stays untouched. + Idempotent; safe because nemo-run regenerates scripts on next launch and + we run before any new job is submitted. + """ + import shutil + from pathlib import Path + + import nemo_skills.pipeline.utils as pipeline_utils + + cluster_config = pipeline_utils.get_cluster_config(cluster) + job_dir = cluster_config.get("job_dir") + if not job_dir: + return + root = Path(job_dir) / "experiments" + if not root.is_dir(): + return + for expname in expnames: + target = root / expname + if target.is_dir(): + shutil.rmtree(target, ignore_errors=True) + + +ENRICH_MODULE = "nvflow.lib.sdg.document_grounded.enrich_rollouts" +ENRICH_MODULE_EVALUATE = "nvflow.lib.sdg.document_grounded.enrich_rollouts_evaluate" +ANALYZE_MODULE = "nvflow.lib.sdg.document_grounded.analyze_rollouts" + + +def submit_gym_generation( + *, + cluster: str, + rollout_expname: str, + run_after: list[str] | None, + input_file: str, + output_dir: str, + prompt_template: str, + gym_path: str, + gym_config_paths: list[str], + gym_agent_name: str, + container: str, + installation_command: str | None, + model_path: str, + num_gpus: int, + server_nodes: int = 1, + num_chunks: int = 1, + num_random_seeds: int = 1, + inference_params: dict[str, Any] | None = None, + vllm_extra: dict[str, Any] | None = None, + extra_record_fields: dict[str, Any] | None = None, + extra_record_field_mappers: dict[str, str] | None = None, + enrich_module: str = ENRICH_MODULE, + rerun_done: bool = False, + gym_uv_venv_dir: str = "", +) -> None: + """Render SDG JSONL to Responses API, then collect rollouts via ``rollout()``. + + Up to two jobs are submitted: + + 1. ``{rollout_expname}-render`` (CPU): ``responses_api render_and_convert`` + turns the flat SDG input into Responses-API rows (per-row prompt under + ``responses_create_params.input`` + per-row ``verifier`` from + ``extra_record_fields``). ``inference_params`` are NOT rendered in -- + they are applied by ``rollout()`` as global ``responses_create_params`` + overrides, keeping ``responses_create_params.input`` stable so the + content-hash join in ``enrich`` matches input<->output rows. + SKIPPED when the render output already exists (unless ``rerun_done``): + re-rendering on resume is wasteful and races a resumed merge's enrich + (see the guard below). When skipped, ``rollout()`` inherits the render's + own ``run_after`` so downstream ordering is preserved. + 2. ``rollout()`` (GPU): chunk + ng_collect_rollouts + per-seed merge, then + the merge job runs ``enrich`` (restore SDG fields + extract generation) + and ``analyze`` (sync ``rollout/output-rs*.jsonl`` up to ``output_dir/``). + + The caller is responsible for any per-stage trim / postprocess, submitted + as a separate ``run_cmd`` under the *stage* expname with + ``run_after=[rollout_expname]`` (so downstream ``run_after=[stage_expname]`` + waits for trim -> rollout). + """ + from nemo_skills.pipeline.cli import run_cmd, wrap_arguments + + from nvflow.core import console + from nvflow.lib.rl.helpers import resolve_host_path + from nvflow.lib.rl.rollout import rollout + + rapi_file = f"{output_dir}/.responses_api_input.jsonl" + render_expname = f"{rollout_expname}-render" + + render_cmd_parts = [ + "python -m nvflow.lib.sdg.document_grounded.responses_api render_and_convert", + f"--input_file {shlex.quote(input_file)}", + f"--output_file {shlex.quote(rapi_file)}", + f"--prompt_template {shlex.quote(prompt_template)}", + ] + if extra_record_fields: + payload = json.dumps(extra_record_fields) + render_cmd_parts.append(f"--extra_record_fields {shlex.quote(payload)}") + if extra_record_field_mappers: + payload = json.dumps(extra_record_field_mappers) + render_cmd_parts.append(f"--extra_record_field_mappers {shlex.quote(payload)}") + render_cmd = " ".join(render_cmd_parts) + + # Skip re-rendering when the Responses-API input already exists. The render + # is a deterministic 1:1 transform of *input_file*, so recomputing it on a + # resume is pure waste (100s of GB rewrite). It is also unsafe: the per-seed + # merge's enrich() reads THIS exact file, and when a seed's chunks are all + # `.done` the merge loses its (transitive, via chunk jobs) dependency on the + # render -- it then runs immediately and can race a concurrent render rewrite, + # reading a half-written file (enrich alignment-check failure). Skipping the + # render keeps the input stable for any resumed merge. Mirrors the + # skip-if-exists guards on the q-prep / q-verify-prep steps; `rerun_done` + # forces a fresh render, kept in lock-step with the rollout rerun. + # NOTE: execute() runs on the orchestrator node, so resolve the container + # path to its host path before checking existence. + rapi_host = resolve_host_path(rapi_file) + rapi_exists = rapi_host.exists() and rapi_host.stat().st_size > 0 + if rapi_exists and not rerun_done: + console.success("Render skipped (reusing existing Responses-API input)") + console.detail("Responses-API input", rapi_file) + rollout_run_after = run_after + else: + run_cmd( + ctx=wrap_arguments(render_cmd), + cluster=cluster, + expname=render_expname, + log_dir=f"{output_dir}/render-logs", + run_after=run_after, + ) + rollout_run_after = [render_expname] + + cfg = build_rollout_config( + input_file=rapi_file, + output_dir=output_dir, + gym_path=gym_path, + gym_config_paths=gym_config_paths, + gym_agent_name=gym_agent_name, + container=container, + installation_command=installation_command, + model_path=model_path, + num_gpus=num_gpus, + server_nodes=server_nodes, + num_chunks=num_chunks, + num_random_seeds=num_random_seeds, + inference_params=inference_params, + vllm_extra=vllm_extra, + rerun_done=rerun_done, + gym_uv_venv_dir=gym_uv_venv_dir, + ) + rollout( + config=cfg, + cluster=cluster, + expname=rollout_expname, + run_after=rollout_run_after, + enrich_module=enrich_module, + analyze_module=ANALYZE_MODULE, + ) + + +def parse_stage_kwargs(stage_kwargs: dict[str, Any]) -> dict[str, Any]: + """Extract normalized fields from a legacy ``args`` / ``ctx_args`` block. + + Returns a dict with ``model_path``, ``num_gpus``, ``server_nodes``, + ``num_chunks``, ``num_random_seeds``, ``prompt_template``, + ``generation_key``, ``inference_params`` and ``vllm_extra`` (any remaining + ``args`` keys that are vLLM serve flags). Used by the generate_* shims to + feed both the render step (prompt_template) and :func:`build_rollout_config`. + """ + args = stage_kwargs.get("args", {}).copy() + ctx_args = stage_kwargs.get("ctx_args", "") + + model_path = args.pop("model", "") + num_gpus = args.pop("server_gpus", args.pop("num_gpus", 8)) + server_nodes = args.pop("server_nodes", 1) + num_chunks = args.pop("num_chunks", 1) + num_random_seeds = args.pop("num_random_seeds", 1) + args.pop("server_type", None) + args.pop("skip_filled", None) + + prompt_template = "" + generation_key = "generation" + inference_params: dict[str, Any] = {} + for part in ctx_args.split(): + if part.startswith("++prompt_config="): + prompt_template = part.split("=", 1)[1] + elif part.startswith("++inference."): + key = part.split("=")[0].replace("++inference.", "") + val = part.split("=", 1)[1] + try: + inference_params[key] = float(val) + except ValueError: + inference_params[key] = val + elif part.startswith("++generation_key="): + generation_key = part.split("=", 1)[1] + + vllm_extra = {k: v for k, v in args.items() if k != "generation_key"} + + return { + "model_path": model_path, + "num_gpus": num_gpus, + "server_nodes": server_nodes, + "num_chunks": num_chunks, + "num_random_seeds": num_random_seeds, + "prompt_template": prompt_template, + "generation_key": generation_key, + "inference_params": inference_params, + "vllm_extra": vllm_extra, + } + + +def build_rollout_config( + *, + input_file: str, + output_dir: str, + gym_path: str, + gym_config_paths: list[str], + gym_agent_name: str, + container: str, + installation_command: str | None, + model_path: str, + num_gpus: int, + server_nodes: int = 1, + num_chunks: int = 1, + num_random_seeds: int = 1, + inference_params: dict[str, Any] | None = None, + vllm_extra: dict[str, Any] | None = None, + rerun_done: bool = False, + env_key: str = "sdg_format_verification", + gym_uv_venv_dir: str = "", +) -> dict[str, Any]: + """Translate SDG generation params into a config for ``rollout()``. + + ``rollout()`` is reused unmodified (the adapter lives entirely on the SDG + side). Notes: + + - ``input_file`` MUST already be in Responses API format (per-row + ``responses_create_params.input`` + per-row ``verifier``), produced by + ``responses_api.render_and_convert``. The per-row prompt and verifier + live in the data, NOT here. + - ``inference_params`` (temperature, top_p, max_output_tokens, ...) become + global ``responses_create_params`` overrides applied by ng_collect. + - No ``judge_vllm`` is set -> ``determine_judge_mode`` returns + ``policy_as_judge`` (no judge server). + - ``environments`` carries the SDG overlay; ``build_config_paths_str`` + prepends the vLLM model config automatically. + """ + # Some knobs are rollout-level (consumed by ``rollout()``), not vLLM serve + # flags, but they arrive mixed into ``vllm_extra`` from a stage's ``args`` / + # ``policy_vllm`` block. Intercept them here so they reach the ``rollout`` + # config instead of leaking into ``policy_vllm`` -> ``build_vllm_server_args`` + # as invalid CLI flags. + # - num_samples_in_parallel: concurrent requests per server (default 4). + # - dependent_jobs: chained resume jobs per chunk so a rollout that doesn't + # finish inside the Slurm walltime continues in the next chained job + # (default 0). Essential for big/slow models where one 4h job can't + # finish (long-tail generations) -- the chained job resumes the few + # remaining samples and exits early once done. + rollout_level_keys = ("num_samples_in_parallel", "dependent_jobs") + extra = dict(vllm_extra or {}) + rollout_level = {k: extra.pop(k) for k in rollout_level_keys if k in extra} + + policy_vllm: dict[str, Any] = { + "model_path": model_path, + "num_gpus": num_gpus, + "server_nodes": server_nodes, + } + policy_vllm.update(extra) + + rollout_cfg: dict[str, Any] = { + "input_data": input_file, + "policy_vllm": policy_vllm, + "responses_create_params": inference_params or {}, + "num_chunks": num_chunks, + "num_random_seeds": num_random_seeds, + "rerun_done": rerun_done, + } + rollout_cfg.update(rollout_level) + + return { + "output_dir": output_dir, + "gym_path": gym_path, + "gym_uv_venv_dir": gym_uv_venv_dir, + "container": container, + "installation_command": installation_command, + "rollout": rollout_cfg, + "environments": { + env_key: { + "agent_name": gym_agent_name, + "config_paths": list(gym_config_paths), + } + }, + } + + +def build_trim_cmd( + *, + stage_name: str, + paths: list[str], + domain_keep_fields: list[str] | None, + extra_keep_fields: list[str] | None = None, +) -> str: + """Build the shell command that trims this stage's output JSONL files. + + The returned string invokes ``nvflow.generic_stage.sdg.document_grounded._trim_cli`` + with the keep-list ``(STAGE_KEEP[stage_name] | domain_keep_fields - + ALWAYS_DROP) | extra_keep_fields`` and the given ``paths`` (files, + directories, or globs -- the CLI expands them). + + ``extra_keep_fields`` is unioned *after* the ``ALWAYS_DROP`` subtraction, so + it is the only way to retain a field that is otherwise in ``ALWAYS_DROP`` + (e.g. ``responses_create_params`` on the final ``dgsdg_post_process`` + output, where the Responses-API original form must survive). Use sparingly. + + The command is meant to be either: + - appended to ``postprocess_cmd`` for Gym-driven stages + (``sdg_generate``-based: question gen/verify, answer gen, genselect, + evaluate), so it runs inside the merge job and the producing stage's + advertised expname does not need to change; or + - chained via ``&&`` to the stage's main CPU command for non-Gym stages + (aggregate, difficulty aggregate, post-process). + + Either way the trim is guaranteed to finish before any downstream stage's + ``run_after`` clears, with zero extra Slurm overhead. + """ + if stage_name not in STAGE_KEEP: + raise KeyError( + f"build_trim_cmd: stage {stage_name!r} is not in STAGE_KEEP. " + f"Known stages: {sorted(STAGE_KEEP)}" + ) + domain = set(domain_keep_fields or []) + keep = ((STAGE_KEEP[stage_name] | domain) - ALWAYS_DROP) | set(extra_keep_fields or []) + keep_args = " ".join(sorted(keep)) + paths_arg = " ".join(shlex.quote(p) for p in paths) + return ( + "python -m nvflow.generic_stage.sdg.document_grounded._trim_cli " + f"--paths {paths_arg} --keep_fields {keep_args}" + ) diff --git a/nvflow/generic_stage/sdg/document_grounded/_schemas.py b/nvflow/generic_stage/sdg/document_grounded/_schemas.py new file mode 100644 index 0000000..6631ce5 --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/_schemas.py @@ -0,0 +1,181 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Per-stage JSONL field allowlists for DG-SDG. + +Each generic DG-SDG stage projects its output JSONL to ``STAGE_KEEP[stage] | +domain_keep_fields`` (set union) at the stage boundary, before the next stage +reads it. Goal: drop stale fields that would silently contaminate downstream +stages -- most importantly the NeMo-Gym rollout metadata and the ``generation`` +/ ``reasoning_content`` keys that get overwritten by every Gym call. + +Domain-specific fields (e.g. ``company_name``, ``file_path0``) are supplied +per recipe via the workflow YAML key ``domain_keep_fields`` and unioned with +``STAGE_KEEP[stage]`` at trim time. Generic stage code never hardcodes them. +""" + +# Cross-stage scratch / noise that we *never* want to survive a stage boundary. +# These are always dropped on top of (i.e. removed from) the per-stage KEEP +# allowlist so that even if a future contributor adds one to STAGE_KEEP by +# mistake, the trim still filters it out. +ALWAYS_DROP: frozenset[str] = frozenset( + { + # NeMo-Gym rollout passthrough metadata (added by responses_api on every + # Gym call; never read downstream). + "_ng_task_index", + "_ng_rollout_index", + "agent_ref", + "reward", + "match_details", + "verifier", + # Generation-time bookkeeping added by responses_api / Gym workers. + "serialized_output", + "num_generated_tokens", + "finish_reason", + "generation_start_time", + "generation_end_time", + "generation_time", + "responses_create_params", + } +) + + +# Per-stage allowlist of *generic* fields (i.e. fields that the lib code +# produces or that downstream lib code needs). Domain-specific fields come +# from the workflow YAML's ``domain_keep_fields`` and are unioned at trim time. +# +# Stage 0 (``dg_sdg_preprocess``) is intentionally absent: it manufactures the +# initial JSONL from raw documents, so there is no upstream record to project +# from. The Stage 1 trim acts as the safety net if the recipe writes junk. +STAGE_KEEP: dict[str, frozenset[str]] = { + # Q-side output (``verified/output-rs*.jsonl``): keep the Yes/No + # ``generation`` because the A-prep step votes on it; drop the Q-verify + # CoT (``reasoning_content``) -- nobody downstream reads it. + "generate_verified_questions": frozenset( + { + "context", + "problem", + "question_type", + "generation", + } + ), + # A-side output (``generated/output-rs*.jsonl``): keep the answer text + # (``generation``) and the answer CoT (``reasoning_content``); both get + # snapshotted into ``reference_*`` by genselect.postprocess in Stage 3. + # + # ``answer_response`` / ``answer_responses_create_params`` carry the *full* + # Responses-API original form of each candidate answer (the exact request + + # response object the A-gen model produced). They are the literal + # ``response`` / ``responses_create_params`` snapshotted under a non- + # ALWAYS_DROP alias by ``enrich_rollouts`` so the trim keeps them. + # genselect collapses the per-seed ``answer_response`` into + # ``answer_responses_list`` and selects one into ``reference_response`` for + # the final post-process output (Responses-API ``final_result.jsonl``). + "generate_answers": frozenset( + { + "context", + "problem", + "question_type", + "question_voting_pass_rate", + "question_voting_total", + "generation", + "reasoning_content", + "answer_response", + "answer_responses_create_params", + } + ), + # GenSelect-picked output (``selected_answers.jsonl``): ``reference_*`` + # carry the selected answer through evaluate/aggregate/difficulty; + # ``generation`` carries the same selected answer as the prompt input for + # evaluate. Genselect scaffolding (solutions/generations_list/answer_N/...) + # is dropped because it has served its purpose. + "gym_genselect_answers": frozenset( + { + "context", + "problem", + "question_type", + "question_voting_pass_rate", + "question_voting_total", + "reference_answer", + "reference_reasoning", + "reference_response", + "reference_responses_create_params", + "generation", + "genselect_answers_metadata", + } + ), + # Multi-seed eval rollouts: keep ``evaluate_generation`` for aggregate to + # parse; drop ``reasoning_content`` which by now is the evaluate-judge CoT + # (not the answer reasoning) and would otherwise silently overwrite the + # real answer CoT carried in ``reference_reasoning``. + "evaluate_answers": frozenset( + { + "context", + "problem", + "question_type", + "question_voting_pass_rate", + "question_voting_total", + "reference_answer", + "reference_reasoning", + "reference_response", + "reference_responses_create_params", + "generation", + "evaluate_generation", + } + ), + # Aggregated answers: per-seed ``evaluate_generation`` and ``correct`` are + # dropped; the consensus ``answerable`` survives. + "aggregate_answers": frozenset( + { + "context", + "problem", + "question_type", + "question_voting_pass_rate", + "question_voting_total", + "reference_answer", + "reference_reasoning", + "reference_response", + "reference_responses_create_params", + "generation", + "answerable", + } + ), + # Final training data (``final_result.jsonl``): post-process has already + # renamed ``reference_reasoning -> reasoning_content`` and + # ``reference_answer -> answer``, so the allowlist uses the post-rename + # names. ``genselect_answers_metadata`` is intentionally dropped from the + # final output -- it was useful for debugging mid-pipeline but is noise + # for SFT / RL. + # ``response`` + ``responses_create_params`` are the Responses-API original form + # post-process restores (renamed from ``reference_response`` / + # ``reference_responses_create_params``); ``expected_answer`` mirrors + # ``answer``. Note ``responses_create_params`` is in ALWAYS_DROP, so the + # post-process stage re-adds it via ``build_trim_cmd(extra_keep_fields=...)`` + # -- listing it here is documentation; the trim would otherwise strip it. + "dgsdg_post_process": frozenset( + { + "context", + "problem", + "answer", + "reasoning_content", + "question_type", + "answerable", + "question_voting_pass_rate", + "question_voting_total", + "expected_answer", + "response", + "responses_create_params", + } + ), +} diff --git a/nvflow/generic_stage/sdg/document_grounded/_trim_cli.py b/nvflow/generic_stage/sdg/document_grounded/_trim_cli.py new file mode 100644 index 0000000..91db185 --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/_trim_cli.py @@ -0,0 +1,121 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Trim DG-SDG JSONL files in place to a per-stage allowlist. + +Invoked at every DG-SDG stage boundary (either as part of the producing job's +``postprocess_cmd`` for Gym stages, or chained with ``&&`` to the CPU command +for non-Gym stages). Drops every JSON key not present in ``--keep_fields``, +including the cross-stage scratch listed in ``_schemas.ALWAYS_DROP``. + +The trim is in-place via a ``.trim_tmp`` rename, so partial failures +don't leave a half-written file at the canonical path. + +Usage:: + + python -m nvflow.generic_stage.sdg.document_grounded._trim_cli \\ + --paths /abs/path/to/file.jsonl /abs/path/to/dir \\ + --keep_fields context problem generation +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def trim_file(path: Path, keep: set[str]) -> tuple[int, int]: + """Rewrite ``path`` in place keeping only top-level keys in ``keep``. + + Returns ``(records_processed, field_instances_dropped)``. + """ + tmp = path.with_suffix(path.suffix + ".trim_tmp") + rec_count = 0 + drop_count = 0 + with path.open() as fin, tmp.open("w") as fout: + for line in fin: + stripped = line.strip() + if not stripped: + continue + record = json.loads(stripped) + slim = {k: v for k, v in record.items() if k in keep} + drop_count += len(record) - len(slim) + fout.write(json.dumps(slim) + "\n") + rec_count += 1 + tmp.replace(path) + return rec_count, drop_count + + +def _resolve_paths(args_paths: list[str]) -> list[Path]: + """Expand globs and directories into a flat list of JSONL files.""" + matched: list[Path] = [] + for raw in args_paths: + candidate = Path(raw) + if "*" in raw or "?" in raw: + matched.extend(sorted(candidate.parent.glob(candidate.name))) + elif candidate.is_dir(): + # Unlike shell globs, pathlib's glob("*.jsonl") also matches + # dotfiles (e.g. ``.responses_api_input.jsonl``, the internal + # render/join cache written by responses_api.render_and_convert). + # That file is never a stage *output* -- trimming it strips + # ``responses_create_params`` (ALWAYS_DROP), which enrich_rollouts' + # join key is computed from, silently poisoning the cache for any + # future re-merge. Exclude dotfiles to match intended shell-glob + # semantics and keep internal caches out of stage-boundary trims. + matched.extend( + sorted(p for p in candidate.glob("*.jsonl") if not p.name.startswith(".")) + ) + else: + matched.append(candidate) + return matched + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--paths", + nargs="+", + required=True, + help="JSONL files, directories (globbed as *.jsonl), or glob patterns.", + ) + parser.add_argument( + "--keep_fields", + nargs="+", + required=True, + help="Top-level JSON keys to keep. Everything else is dropped.", + ) + args = parser.parse_args(argv) + + keep = set(args.keep_fields) + files = _resolve_paths(args.paths) + if not files: + print( + f"[trim] no files matched from {args.paths!r}; nothing to do", + file=sys.stderr, + ) + return 0 + + for f in files: + if not f.exists(): + print(f"[trim] {f}: missing, skipping", file=sys.stderr) + continue + n, d = trim_file(f, keep) + print(f"[trim] {f}: {n} records, dropped {d} field-instances") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/nvflow/recipes/finance/stages/sdg/aggregate_answers.py b/nvflow/generic_stage/sdg/document_grounded/aggregate_answers.py similarity index 66% rename from nvflow/recipes/finance/stages/sdg/aggregate_answers.py rename to nvflow/generic_stage/sdg/document_grounded/aggregate_answers.py index 7f39179..d759e44 100644 --- a/nvflow/recipes/finance/stages/sdg/aggregate_answers.py +++ b/nvflow/generic_stage/sdg/document_grounded/aggregate_answers.py @@ -17,27 +17,13 @@ from pathlib import Path from typing import Any -from nvflow.core import BaseStage, StageRegistry, console +from nvflow.core import BaseStage, console +from ._helpers import build_trim_cmd -@StageRegistry.register( - recipe="finance", - workflow="document_grounded_sdg", - stage="aggregate_answers", -) -class AggregateAnswersStage(BaseStage): - """Aggregate multi-seed evaluation results. - - This stage processes output-rs*.jsonl files in streaming mode: - - Reads all seed files line-by-line in parallel (no intermediate files) - - Parses evaluate_generation inline - - Only keeps records where ALL seeds have correct=YES - - Only keeps records where ALL seeds have consistent answerable (all YES or all NO) - - Adds a final 'answerable' field based on the consistent value - This ensures high-quality data where the evaluation is confident and consistent - across multiple random samples. Uses O(1) memory regardless of file size. - """ +class AggregateAnswersStage(BaseStage): + """Aggregate multi-seed evaluation results.""" workflow = "document_grounded_sdg" @@ -61,22 +47,21 @@ def execute( console.detail("Num seeds", str(num_seeds)) console.blank() - # The evaluate_answers stage creates: {input_dir}/{input_file_stem}/output-rsN.jsonl - # Input file stem is "selected_answers" based on workflow config generation_folder = Path(input_dir) / "selected_answers" - - aggregate_module = "nvflow.recipes.finance.utils.sdg.aggregate_evaluate" - - # Aggregate results (parse + aggregate combined, no intermediate files) - full_cmd = ( - f"python3 -m {aggregate_module} " + aggregate_cmd = ( + "python -m nvflow.lib.sdg.document_grounded.aggregate " f"--input_dir {generation_folder} " f"--output_file {output_file} " f"--num_seeds {num_seeds}" ) + trim_cmd = build_trim_cmd( + stage_name="aggregate_answers", + paths=[output_file], + domain_keep_fields=config.get("domain_keep_fields"), + ) + full_cmd = f"{aggregate_cmd} && {trim_cmd}" console.status("Running aggregation (streaming, no intermediate files)") - run_cmd( ctx=wrap_arguments(full_cmd), cluster=cluster, diff --git a/nvflow/recipes/finance/stages/sdg/dg_sdg_preprocess.py b/nvflow/generic_stage/sdg/document_grounded/dg_sdg_preprocess.py similarity index 50% rename from nvflow/recipes/finance/stages/sdg/dg_sdg_preprocess.py rename to nvflow/generic_stage/sdg/document_grounded/dg_sdg_preprocess.py index c0ada3f..5e4bea6 100644 --- a/nvflow/recipes/finance/stages/sdg/dg_sdg_preprocess.py +++ b/nvflow/generic_stage/sdg/document_grounded/dg_sdg_preprocess.py @@ -12,32 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""SEC Data Preprocessing Stage for Document-Grounded SDG. - -This stage processes raw SEC filings (10-K and 10-Q HTML files) into structured JSONL data: -1. Chunk HTML files into Markdown, Clean HTML, and Original HTML -2. Generate CSV file lists from chunked files -3. Generate JSONL training data from CSVs -""" +"""Data preprocessing stage for Document-Grounded SDG.""" from typing import Any -from nvflow.core import BaseStage, StageRegistry, console +from nvflow.core import BaseStage, console +from nvflow.lib.rl.helpers import resolve_host_path -@StageRegistry.register( - recipe="finance", - workflow="document_grounded_sdg", - stage="dg_sdg_preprocess", -) class DGSDGPreprocessStage(BaseStage): - """Preprocess SEC filings for document-grounded SDG. - - This stage converts raw SEC HTML filings into structured JSONL data: - 1. Chunks HTML files by token count with overlap - 2. Generates CSV file lists for tracking chunks - 3. Creates JSONL training data with proper sampling distribution - """ + """Preprocess domain documents into structured JSONL data.""" workflow = "document_grounded_sdg" @@ -48,34 +32,70 @@ def execute( expname: str, run_after: list[str] | None = None, ) -> None: - """Execute the SEC data preprocessing pipeline.""" + """Execute the data preprocessing pipeline.""" from nemo_skills.pipeline.cli import run_cmd, wrap_arguments input_dir = config["input_dir"] output_dir = config["output_dir"] distribution_dir = config["distribution_dir"] - # Chunking settings max_tokens = config.get("max_tokens", 2000) overlap_tokens = config.get("overlap_tokens", 100) - - # Sampling settings total_samples = config.get("total_samples", 150000) max_skip_count = config.get("max_skip_count", 20000) seed = config.get("seed", 42) - - console.status("SEC Data Preprocessing") + preprocess_module = config["preprocess_module"] + rerun_done = config.get("rerun_done", False) + + # Domain-agnostic passthrough: arbitrary extra CLI args forwarded verbatim + # to the preprocess_module. Lets domain recipes pass module-specific flags + # (e.g. the SEC recipe's --forms) without this generic stage knowing about + # them. Bool True -> bare flag; other values -> "--key value" (quoted). + extra_args = config.get("extra_args") or {} + extra_parts: list[str] = [] + for key, value in extra_args.items(): + if isinstance(value, bool): + if value: + extra_parts.append(f"--{key}") + elif isinstance(value, list | tuple): + extra_parts.append(f"--{key} '{' '.join(str(v) for v in value)}'") + elif isinstance(value, str): + extra_parts.append(f"--{key} '{value}'") + else: + extra_parts.append(f"--{key} {value}") + extra_args_str = " ".join(extra_parts) + + console.status("Document data preprocessing") console.detail("Input dir", input_dir) console.detail("Output dir", output_dir) console.detail("Distribution dir", distribution_dir) + console.detail("Preprocess module", preprocess_module) console.detail("Max tokens", str(max_tokens)) console.detail("Overlap tokens", str(overlap_tokens)) console.detail("Total samples", str(total_samples)) console.detail("Max skip count", str(max_skip_count)) console.detail("Seed", str(seed)) + if extra_args_str: + console.detail("Extra args", extra_args_str) console.blank() - preprocess_module = "nvflow.recipes.finance.utils.sdg.dg_sdg_data_preprocess" + # Reuse previously materialized sampling output by default. + # Set rerun_done=true to force a full regenerate. + # + # ``execute()`` runs on the orchestrator/login node, so ``output_dir`` + # (a container path like ``/workspace/...``) must be resolved to its + # host path before the existence check -- otherwise it never matches and + # sampling re-runs on every launch. + forms_arg = str((extra_args or {}).get("forms", "10-K 10-Q")) + forms = [f for f in forms_arg.split() if f] + host_jsonl_dir = resolve_host_path(f"{output_dir}/jsonl") + if forms and not rerun_done: + expected_outputs = [host_jsonl_dir / f"{form.lower()}-data.jsonl" for form in forms] + all_present = all(p.exists() and p.stat().st_size > 0 for p in expected_outputs) + if all_present: + console.success("Data preprocessing skipped (reusing existing sampled output)") + console.detail("Output directory", output_dir) + return full_cmd = ( f"python3 -m {preprocess_module} " @@ -88,8 +108,8 @@ def execute( f"--max_skip_count {max_skip_count} " f"--seed {seed}" ) - - console.status("Running SEC data preprocessing") + if extra_args_str: + full_cmd += f" {extra_args_str}" run_cmd( ctx=wrap_arguments(full_cmd), @@ -98,12 +118,12 @@ def execute( run_after=run_after, ) - console.success("SEC data preprocessing job submitted") + console.success("Data preprocessing job submitted") console.detail("Output directory", output_dir) def validate_config(self, config: dict[str, Any]) -> None: """Validate stage configuration.""" - required = ["input_dir", "output_dir", "distribution_dir"] + required = ["input_dir", "output_dir", "distribution_dir", "preprocess_module"] for field in required: if field not in config: raise ValueError(f"Missing required field: {field}") diff --git a/nvflow/recipes/finance/stages/sdg/document_grounded_data.py b/nvflow/generic_stage/sdg/document_grounded/dgsdg_post_process.py similarity index 57% rename from nvflow/recipes/finance/stages/sdg/document_grounded_data.py rename to nvflow/generic_stage/sdg/document_grounded/dgsdg_post_process.py index 360ab0c..8c4c341 100644 --- a/nvflow/recipes/finance/stages/sdg/document_grounded_data.py +++ b/nvflow/generic_stage/sdg/document_grounded/dgsdg_post_process.py @@ -12,32 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Document grounded sdg data post processing stage.""" +"""Document grounded SDG data post processing stage.""" from typing import Any -from nvflow.core import BaseStage, StageRegistry, console +from nvflow.core import BaseStage, console +from ._helpers import build_trim_cmd -@StageRegistry.register( - recipe="finance", - workflow="document_grounded_sdg", - stage="dgsdg_post_process", -) -class DGSDGPostProcessStage(BaseStage): - """Post process document grounded sdg data by cleaning fields and creating subsets. - This stage: - 1. Removes unwanted fields (solutions, generations_list, etc.) - 2. Renames reference_reasoning -> reasoning_content, reference_answer -> answer - 3. Creates full_data.jsonl with all cleaned records - 4. Creates medium_sft_data.jsonl: - - Only records with difficulty_score in [1, 2, 3, 4] - - For 10-K filings: excludes Risk_Factors questions - - For 10-Q filings: only includes Risk_Factors questions - 5. Creates hard_rl_data.jsonl: - - Only records with difficulty_score = 0 - """ +class DGSDGPostProcessStage(BaseStage): + """Post process document grounded SDG data by cleaning fields and creating subsets.""" workflow = "document_grounded_sdg" @@ -48,24 +33,36 @@ def execute( expname: str, run_after: list[str] | None = None, ) -> None: - """Execute document grounded sdg data post processing.""" + """Execute document grounded SDG data post processing.""" from nemo_skills.pipeline.cli import run_cmd, wrap_arguments input_file = config["input_file"] output_dir = config["output_dir"] seed = config.get("seed", 42) + postprocess_script = config["postprocess_script"] - console.status("Post processing document grounded sdg data") + console.status("Post processing document grounded SDG data") console.detail("Input file", input_file) console.detail("Output dir", output_dir) console.detail("Random seed", str(seed)) + console.detail("Postprocess script", postprocess_script) console.blank() - module = "nvflow.recipes.finance.utils.sdg.dgsdg_post_process" - - cmd = ( - f"python3 -m {module} --input_file {input_file} --output_dir {output_dir} --seed {seed}" + postprocess_cmd = ( + f"python {postprocess_script} " + f"--input_file {input_file} " + f"--output_dir {output_dir} " + f"--seed {seed}" + ) + trim_cmd = build_trim_cmd( + stage_name="dgsdg_post_process", + paths=[f"{output_dir}/final_result.jsonl"], + domain_keep_fields=config.get("domain_keep_fields"), + # ``responses_create_params`` is in ALWAYS_DROP; re-add it here so the + # final Responses-API record retains the original request. + extra_keep_fields=["responses_create_params"], ) + cmd = f"{postprocess_cmd} && {trim_cmd}" run_cmd( ctx=wrap_arguments(cmd), @@ -74,12 +71,12 @@ def execute( run_after=run_after, ) - console.success("Document grounded sdg data post processing job submitted") + console.success("Document grounded SDG data post processing job submitted") console.detail("Output files will be in", output_dir) def validate_config(self, config: dict[str, Any]) -> None: """Validate stage configuration.""" - required = ["input_file", "output_dir"] + required = ["input_file", "output_dir", "postprocess_script"] for field in required: if field not in config: raise ValueError(f"Missing required field: {field}") diff --git a/nvflow/generic_stage/sdg/document_grounded/evaluate_answers.py b/nvflow/generic_stage/sdg/document_grounded/evaluate_answers.py new file mode 100644 index 0000000..14556db --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/evaluate_answers.py @@ -0,0 +1,152 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Evaluate answers for correctness and answerability.""" + +from pathlib import Path +from typing import Any + +from nvflow.core import BaseStage, console + +from ._helpers import ( + ENRICH_MODULE_EVALUATE, + build_trim_cmd, + clean_stale_experiments, + submit_gym_generation, +) + + +class EvaluateAnswersStage(BaseStage): + """Evaluate answers for correctness and answerability.""" + + workflow = "document_grounded_sdg" + + def execute( + self, + config: dict[str, Any], + cluster: str, + expname: str, + run_after: list[str] | None = None, + ) -> None: + """Execute answer evaluation and filtering.""" + from nemo_skills.pipeline.cli import run_cmd, wrap_arguments + + clean_stale_experiments(cluster, [f"{expname}-gen", f"{expname}-gen-render", expname]) + + input_file = config["input_file"] + output_dir = config.get("output_dir") + output_file = config.get("output_file") + prompt_template = config.get("prompt_template", config.get("prompt_config", "")) + generation_key = config.get("generation_key", "evaluate_generation") + inference_params = config.get("inference_params", {}) + num_random_seeds = config.get("num_random_seeds", 1) + + if generation_key != "evaluate_generation": + console.warning( + "evaluate_answers currently pins generation field to " + "'evaluate_generation' (rollout enrich hook is fixed-arg); " + f"configured generation_key='{generation_key}' is ignored." + ) + + console.status("Evaluating answers for correctness and answerability (NeMo-Gym)") + console.detail("Input file", input_file) + console.detail("Output dir", str(output_dir)) + console.detail("Prompt template", prompt_template) + console.detail("Num random seeds", str(num_random_seeds)) + console.blank() + + if output_dir: + generation_folder = Path(output_dir) / Path(input_file).stem + else: + generation_folder = Path(output_file).parent / Path(input_file).stem + + console.detail("Generation folder", str(generation_folder)) + + lib_evaluate = "python -m nvflow.lib.sdg.document_grounded.evaluate" + domain_keep_fields = config.get("domain_keep_fields") + + pv = dict(config.get("policy_vllm", {})) + model_path = pv.pop("model_path", "") + num_gpus = pv.pop("num_gpus", 8) + server_nodes = pv.pop("server_nodes", 1) + + console.status("Running LLM evaluation via NeMo-Gym") + gen_expname = f"{expname}-gen" + submit_gym_generation( + cluster=cluster, + rollout_expname=gen_expname, + run_after=run_after, + input_file=input_file, + output_dir=str(generation_folder), + prompt_template=prompt_template, + gym_path=config["gym_path"], + gym_config_paths=config.get("gym_config_paths", []), + gym_agent_name=config["gym_agent_name"], + container=config.get("container", "nemo-rl"), + installation_command=config.get("installation_command"), + gym_uv_venv_dir=config.get("gym_uv_venv_dir", ""), + model_path=model_path, + num_gpus=num_gpus, + server_nodes=server_nodes, + num_chunks=config.get("num_chunks", 1), + num_random_seeds=num_random_seeds, + inference_params=inference_params, + vllm_extra=pv, + extra_record_fields=config.get("extra_record_fields"), + extra_record_field_mappers=config.get("extra_record_field_mappers"), + enrich_module=ENRICH_MODULE_EVALUATE, + rerun_done=config.get("rerun_done", False), + ) + + # Parse/filter/trim (single-seed) or trim-only (multi-seed), run under + # the stage expname so downstream `run_after=[stage_expname]` waits. + if num_random_seeds <= 1: + generated_file = str(generation_folder / "output-rs0.jsonl") + parsed_file = str(generation_folder / "parsed.jsonl") + final_output = ( + output_file if output_file else str(generation_folder / "evaluated.jsonl") + ) + parse_cmd = ( + f"{lib_evaluate} parse --input_file {generated_file} --output_file {parsed_file}" + ) + filter_cmd = ( + f"{lib_evaluate} filter --input_file {parsed_file} --output_file {final_output}" + ) + trim_cmd = build_trim_cmd( + stage_name="evaluate_answers", + paths=[final_output], + domain_keep_fields=domain_keep_fields, + ) + postprocess_cmd = f"{parse_cmd} && {filter_cmd} && {trim_cmd}" + else: + postprocess_cmd = build_trim_cmd( + stage_name="evaluate_answers", + paths=[str(generation_folder)], + domain_keep_fields=domain_keep_fields, + ) + run_cmd( + ctx=wrap_arguments(postprocess_cmd), + cluster=cluster, + expname=expname, + log_dir=f"{generation_folder}/postprocess-logs", + run_after=[gen_expname], + ) + + console.success(f"Completed Answer Evaluation for: {input_file}") + if num_random_seeds > 1: + console.detail("Parsed outputs in", str(generation_folder)) + else: + console.detail( + "Output (correct answers only, with 'answerable' field)", str(output_file) + ) diff --git a/nvflow/generic_stage/sdg/document_grounded/generate_answers.py b/nvflow/generic_stage/sdg/document_grounded/generate_answers.py new file mode 100644 index 0000000..f0d2104 --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/generate_answers.py @@ -0,0 +1,195 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Answer generation pipeline for document-grounded SDG. + +Consumes verified-question records produced by GenerateVerifiedQuestionsStage +and emits N candidate answers per question for downstream genselect. +""" + +from typing import Any + +from nvflow.core import BaseStage, console +from nvflow.lib.rl.helpers import resolve_host_path + +from ._helpers import ( + build_trim_cmd, + clean_stale_experiments, + parse_stage_kwargs, + submit_gym_generation, +) + + +class GenerateAnswersStage(BaseStage): + """A-side of DG-SDG: a-prep (threshold filter) -> A-gen. + + Output layout under ``output_dir``:: + + answer_input.jsonl # step 1 output (questions surviving the + # verification threshold) + generated/ # step 2 output (A-gen rollouts; consumed by + # gym_genselect_answers) + """ + + workflow = "document_grounded_sdg" + + def execute( + self, + config: dict[str, Any], + cluster: str, + expname: str, + run_after: list[str] | None = None, + ) -> None: + from nemo_skills.pipeline.cli import run_cmd, wrap_arguments + + clean_stale_experiments( + cluster, + [ + f"{expname}-step1-a-prep", + f"{expname}-step2-a-gen", + f"{expname}-step2-a-gen-render", + expname, + ], + ) + + input_dir = config["input_dir"] + output_dir = config["output_dir"] + + gym_path = config["gym_path"] + gym_uv_venv_dir = config.get("gym_uv_venv_dir", "") + gym_config_paths_default = config.get("gym_config_paths", []) + gym_agent_name_default = config.get("gym_agent_name") + gym_container = config.get("container", "nemo-rl") + installation_command = config.get("installation_command") + extra_record_fields_default = config.get("extra_record_fields") + extra_record_field_mappers_default = config.get("extra_record_field_mappers") + + def _substep(prefix: str) -> dict[str, Any]: + agent = config.get(f"{prefix}_gym_agent_name", gym_agent_name_default) + if not agent: + raise ValueError( + f"generate_answers: '{prefix}_gym_agent_name' " + "(or stage-level 'gym_agent_name') is required." + ) + return { + "gym_config_paths": config.get( + f"{prefix}_gym_config_paths", gym_config_paths_default + ), + "gym_agent_name": agent, + "extra_record_fields": config.get( + f"{prefix}_extra_record_fields", extra_record_fields_default + ), + "extra_record_field_mappers": config.get( + f"{prefix}_extra_record_field_mappers", + extra_record_field_mappers_default, + ), + } + + a_gen_overrides = _substep("answer_generation") + + answer_preprocess_kwargs = config.get("answer_preprocess_kwargs", {}) + answer_generation_kwargs = config.get("answer_generation_kwargs", {}) + + a_generate_input_file = f"{output_dir}/answer_input.jsonl" + a_generate_output_dir = f"{output_dir}/generated" + + lib_preprocess = "python -m nvflow.lib.sdg.document_grounded.preprocess" + + # execute() runs on the orchestrator node: resolve the container path to + # its host path before checking existence (see _helpers.host_path). + step1_expname = f"{expname}-step1-a-prep" + rerun_a_prep = config.get("answer_prep_rerun_done", False) + a_prep_host = resolve_host_path(a_generate_input_file) + a_prep_exists = a_prep_host.exists() and a_prep_host.stat().st_size > 0 + a_prep_submitted = False + console.status("Step 1/2: Preparing data for answer generation") + console.detail("Output file", a_generate_input_file) + if a_prep_exists and not rerun_a_prep: + console.success("Step 1 skipped (reusing existing answer_input.jsonl)") + else: + console.detail("Input dir", input_dir) + threshold = answer_preprocess_kwargs.get("threshold", 0.5) + sbatch_kwargs = answer_preprocess_kwargs.get("sbatch_kwargs", "") + cmd = ( + f"{lib_preprocess} construct_answer_generate_input " + f"--input_dir {input_dir} " + f"--output_file {a_generate_input_file} " + f"--threshold {threshold}" + ) + run_cmd( + ctx=wrap_arguments(cmd), + cluster=cluster, + expname=step1_expname, + run_after=run_after, + sbatch_kwargs=sbatch_kwargs, + ) + a_prep_submitted = True + console.success("Step 1 job submitted") + + console.status("Step 2/2: Generating answers") + params = parse_stage_kwargs(answer_generation_kwargs) + a_gen_expname = f"{expname}-step2-a-gen" + submit_gym_generation( + cluster=cluster, + rollout_expname=a_gen_expname, + run_after=[step1_expname] if a_prep_submitted else run_after, + input_file=a_generate_input_file, + output_dir=a_generate_output_dir, + prompt_template=params["prompt_template"], + gym_path=gym_path, + gym_config_paths=a_gen_overrides["gym_config_paths"], + gym_agent_name=a_gen_overrides["gym_agent_name"], + container=gym_container, + installation_command=installation_command, + gym_uv_venv_dir=gym_uv_venv_dir, + model_path=params["model_path"], + num_gpus=params["num_gpus"], + server_nodes=params["server_nodes"], + num_chunks=params["num_chunks"], + num_random_seeds=params["num_random_seeds"], + inference_params=params["inference_params"], + vllm_extra=params["vllm_extra"], + extra_record_fields=a_gen_overrides["extra_record_fields"], + extra_record_field_mappers=a_gen_overrides["extra_record_field_mappers"], + rerun_done=answer_generation_kwargs.get("rerun_done", False), + ) + + # Per-stage trim runs under the *stage* expname (depends on a-gen) so + # downstream `run_after=[stage_expname]` waits for the trimmed output. + trim_cmd = build_trim_cmd( + stage_name="generate_answers", + paths=[a_generate_output_dir], + domain_keep_fields=config.get("domain_keep_fields"), + ) + run_cmd( + ctx=wrap_arguments(trim_cmd), + cluster=cluster, + expname=expname, + log_dir=f"{a_generate_output_dir}/trim-logs", + run_after=[a_gen_expname], + ) + console.success("Step 2 job submitted") + + console.blank() + console.success("Answer generation pipeline jobs submitted") + console.detail("Generated answers will be in", a_generate_output_dir) + + def validate_config(self, config: dict[str, Any]) -> None: + """Validate required configuration fields.""" + required = ["input_dir", "output_dir", "gym_path"] + for field in required: + if field not in config: + raise ValueError(f"Missing required field: {field}") + if "answer_generation_kwargs" not in config: + raise ValueError("Missing required field: answer_generation_kwargs") diff --git a/nvflow/generic_stage/sdg/document_grounded/generate_verified_questions.py b/nvflow/generic_stage/sdg/document_grounded/generate_verified_questions.py new file mode 100644 index 0000000..39a56a9 --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/generate_verified_questions.py @@ -0,0 +1,252 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Question generation + verification pipeline for document-grounded SDG.""" + +from typing import Any + +from nvflow.core import BaseStage, console +from nvflow.lib.rl.helpers import resolve_host_path + +from ._helpers import ( + build_trim_cmd, + clean_stale_experiments, + parse_stage_kwargs, + submit_gym_generation, +) + + +class GenerateVerifiedQuestionsStage(BaseStage): + """Q-side of DG-SDG: prep -> generate -> verify-prep -> verify. + + Output layout under ``output_dir``:: + + generate_input.jsonl # step 1 output + generated/ # step 2 output (Q-gen rollouts) + verify_input.jsonl # step 3 output + verified/ # step 4 output (Q-verify rollouts; consumed by + # the generate_answers stage) + """ + + workflow = "document_grounded_sdg" + + def execute( + self, + config: dict[str, Any], + cluster: str, + expname: str, + run_after: list[str] | None = None, + ) -> None: + from nemo_skills.pipeline.cli import run_cmd, wrap_arguments + + clean_stale_experiments( + cluster, + [ + f"{expname}-step1-q-prep", + f"{expname}-step2-q-gen", + f"{expname}-step2-q-gen-render", + f"{expname}-step3-q-verify-prep", + f"{expname}-step4-q-verify", + f"{expname}-step4-q-verify-render", + expname, + ], + ) + + input_folder = config["input_folder"] + output_dir = config["output_dir"] + question_prep_script = config["question_prep_script"] + + gym_path = config["gym_path"] + gym_uv_venv_dir = config.get("gym_uv_venv_dir", "") + gym_config_paths_default = config.get("gym_config_paths", []) + gym_agent_name_default = config.get("gym_agent_name") + gym_container = config.get("container", "nemo-rl") + installation_command = config.get("installation_command") + extra_record_fields_default = config.get("extra_record_fields") + extra_record_field_mappers_default = config.get("extra_record_field_mappers") + + def _substep(prefix: str) -> dict[str, Any]: + agent = config.get(f"{prefix}_gym_agent_name", gym_agent_name_default) + if not agent: + raise ValueError( + f"generate_verified_questions: '{prefix}_gym_agent_name' " + "(or stage-level 'gym_agent_name') is required." + ) + return { + "gym_config_paths": config.get( + f"{prefix}_gym_config_paths", gym_config_paths_default + ), + "gym_agent_name": agent, + "extra_record_fields": config.get( + f"{prefix}_extra_record_fields", extra_record_fields_default + ), + "extra_record_field_mappers": config.get( + f"{prefix}_extra_record_field_mappers", + extra_record_field_mappers_default, + ), + } + + q_gen_overrides = _substep("question_generation") + q_verify_overrides = _substep("question_verify") + + question_generation_kwargs = config.get("question_generation_kwargs", {}) + question_verify_kwargs = config.get("question_verify_kwargs", {}) + rerun_q_prep = config.get("question_prep_rerun_done", False) + rerun_q_verify_prep = config.get("question_verify_prep_rerun_done", False) + + q_generate_input_file = f"{output_dir}/generate_input.jsonl" + q_generate_output_dir = f"{output_dir}/generated" + q_verify_input_file = f"{output_dir}/verify_input.jsonl" + q_verify_output_dir = f"{output_dir}/verified" + + lib_preprocess = "python -m nvflow.lib.sdg.document_grounded.preprocess" + + step1_expname = f"{expname}-step1-q-prep" + # execute() runs on the orchestrator node: resolve the container path to + # its host path before checking existence (see _helpers.host_path). + step1_host = resolve_host_path(q_generate_input_file) + step1_exists = step1_host.exists() and step1_host.stat().st_size > 0 + step1_submitted = False + if step1_exists and not rerun_q_prep: + console.status("Step 1/4: Preparing data for question generation") + console.detail("Output file", q_generate_input_file) + console.success("Step 1 skipped (reusing existing generate_input.jsonl)") + else: + console.status("Step 1/4: Preparing data for question generation") + console.detail("Input folder", input_folder) + console.detail("Output file", q_generate_input_file) + cmd = ( + f"python {question_prep_script} " + f"--input_folder {input_folder} " + f"--output_file {q_generate_input_file}" + ) + run_cmd( + ctx=wrap_arguments(cmd), + cluster=cluster, + expname=step1_expname, + run_after=run_after, + ) + step1_submitted = True + console.success("Step 1 job submitted") + + console.status("Step 2/4: Generating questions") + q_gen_params = parse_stage_kwargs(question_generation_kwargs) + submit_gym_generation( + cluster=cluster, + rollout_expname=f"{expname}-step2-q-gen", + run_after=[step1_expname] if step1_submitted else run_after, + input_file=q_generate_input_file, + output_dir=q_generate_output_dir, + prompt_template=q_gen_params["prompt_template"], + gym_path=gym_path, + gym_config_paths=q_gen_overrides["gym_config_paths"], + gym_agent_name=q_gen_overrides["gym_agent_name"], + container=gym_container, + installation_command=installation_command, + gym_uv_venv_dir=gym_uv_venv_dir, + model_path=q_gen_params["model_path"], + num_gpus=q_gen_params["num_gpus"], + server_nodes=q_gen_params["server_nodes"], + num_chunks=q_gen_params["num_chunks"], + num_random_seeds=q_gen_params["num_random_seeds"], + inference_params=q_gen_params["inference_params"], + vllm_extra=q_gen_params["vllm_extra"], + extra_record_fields=q_gen_overrides["extra_record_fields"], + extra_record_field_mappers=q_gen_overrides["extra_record_field_mappers"], + rerun_done=question_generation_kwargs.get("rerun_done", False), + ) + console.success("Step 2 job submitted") + + step3_expname = f"{expname}-step3-q-verify-prep" + step3_host = resolve_host_path(q_verify_input_file) + step3_exists = step3_host.exists() and step3_host.stat().st_size > 0 + step3_submitted = False + if step3_exists and not rerun_q_verify_prep: + console.status("Step 3/4: Preparing data for question verification") + console.detail("Output file", q_verify_input_file) + console.success("Step 3 skipped (reusing existing verify_input.jsonl)") + else: + console.status("Step 3/4: Preparing data for question verification") + cmd = ( + f"{lib_preprocess} construct_question_verify_input " + f"--input_dir {q_generate_output_dir} " + f"--output_file {q_verify_input_file}" + ) + run_cmd( + ctx=wrap_arguments(cmd), + cluster=cluster, + expname=step3_expname, + run_after=[f"{expname}-step2-q-gen"], + ) + step3_submitted = True + console.success("Step 3 job submitted") + + console.status("Step 4/4: Verifying questions") + q_verify_params = parse_stage_kwargs(question_verify_kwargs) + q_verify_expname = f"{expname}-step4-q-verify" + submit_gym_generation( + cluster=cluster, + rollout_expname=q_verify_expname, + run_after=[step3_expname] if step3_submitted else [f"{expname}-step2-q-gen"], + input_file=q_verify_input_file, + output_dir=q_verify_output_dir, + prompt_template=q_verify_params["prompt_template"], + gym_path=gym_path, + gym_config_paths=q_verify_overrides["gym_config_paths"], + gym_agent_name=q_verify_overrides["gym_agent_name"], + container=gym_container, + installation_command=installation_command, + gym_uv_venv_dir=gym_uv_venv_dir, + model_path=q_verify_params["model_path"], + num_gpus=q_verify_params["num_gpus"], + server_nodes=q_verify_params["server_nodes"], + num_chunks=q_verify_params["num_chunks"], + num_random_seeds=q_verify_params["num_random_seeds"], + inference_params=q_verify_params["inference_params"], + vllm_extra=q_verify_params["vllm_extra"], + extra_record_fields=q_verify_overrides["extra_record_fields"], + extra_record_field_mappers=q_verify_overrides["extra_record_field_mappers"], + rerun_done=question_verify_kwargs.get("rerun_done", False), + ) + + # Stage trim runs under the stage expname (depends on q-verify) so the + # downstream stage's `run_after=[stage_expname]` waits for trimmed output. + trim_cmd = build_trim_cmd( + stage_name="generate_verified_questions", + paths=[q_verify_output_dir], + domain_keep_fields=config.get("domain_keep_fields"), + ) + run_cmd( + ctx=wrap_arguments(trim_cmd), + cluster=cluster, + expname=expname, + log_dir=f"{q_verify_output_dir}/trim-logs", + run_after=[q_verify_expname], + ) + console.success("Step 4 job submitted") + + console.blank() + console.success("Question generation + verification pipeline jobs submitted") + console.detail("Verified questions will be in", q_verify_output_dir) + + def validate_config(self, config: dict[str, Any]) -> None: + """Validate required configuration fields.""" + required = ["input_folder", "output_dir", "gym_path", "question_prep_script"] + for field in required: + if field not in config: + raise ValueError(f"Missing required field: {field}") + if "question_generation_kwargs" not in config: + raise ValueError("Missing required field: question_generation_kwargs") + if "question_verify_kwargs" not in config: + raise ValueError("Missing required field: question_verify_kwargs") diff --git a/nvflow/generic_stage/sdg/document_grounded/gym_genselect_answers.py b/nvflow/generic_stage/sdg/document_grounded/gym_genselect_answers.py new file mode 100644 index 0000000..df61c6c --- /dev/null +++ b/nvflow/generic_stage/sdg/document_grounded/gym_genselect_answers.py @@ -0,0 +1,155 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generate and select best answers using NeMo-Gym inference.""" + +from typing import Any + +from nvflow.core import BaseStage, console +from nvflow.lib.rl.helpers import resolve_host_path + +from ._helpers import ( + build_trim_cmd, + clean_stale_experiments, + submit_gym_generation, +) + + +class GymGenselectAnswersStage(BaseStage): + """Generate and select best answers via NeMo-Gym collect_rollouts.""" + + workflow = "document_grounded_sdg" + + def execute( + self, + config: dict[str, Any], + cluster: str, + expname: str, + run_after: list[str] | None = None, + ) -> None: + """Execute genselect answer generation via rollout().""" + from nemo_skills.pipeline.cli import run_cmd, wrap_arguments + + clean_stale_experiments( + cluster, + [f"{expname}-prep", f"{expname}-gen", f"{expname}-gen-render", expname], + ) + + input_dir = config["input_dir"] + output_file = config["output_file"] + prompt_template = config["prompt_template"] + + output_dir = output_file.replace(".jsonl", "") + prepped_file = output_dir + "_prepped.jsonl" + + console.status("Generating and selecting best answers (NeMo-Gym)") + console.detail("Input dir", input_dir) + console.detail("Output file", output_file) + console.detail("Prepped file", prepped_file) + console.detail("Output dir", output_dir) + console.detail("Prompt template", prompt_template) + console.blank() + + # execute() runs on the orchestrator node: resolve the container path to + # its host path before checking existence (see _helpers.host_path). + prep_expname = f"{expname}-prep" + rerun_prep = config.get("genselect_prep_rerun_done", False) + prep_host = resolve_host_path(prepped_file) + prep_exists = prep_host.exists() and prep_host.stat().st_size > 0 + prep_submitted = False + console.status("Step 1: Preparing genselect data") + if prep_exists and not rerun_prep: + console.success("Step 1 skipped (reusing existing prepped genselect input)") + else: + run_cmd( + ctx=wrap_arguments( + f"python -m nvflow.lib.sdg.document_grounded.genselect merge " + f"--input_dir={input_dir} --output_file={prepped_file}" + ), + cluster=cluster, + expname=prep_expname, + log_dir=f"{output_dir}/prep-data-logs", + run_after=run_after, + ) + prep_submitted = True + + pv = dict(config.get("policy_vllm", {})) + model_path = pv.pop("model_path", "") or config.get("model", "") + num_gpus = pv.pop("num_gpus", 0) or config.get("server_gpus", 8) + server_nodes = pv.pop("server_nodes", 1) + + console.status("Step 2: Generating answers via NeMo-Gym") + gen_expname = f"{expname}-gen" + submit_gym_generation( + cluster=cluster, + rollout_expname=gen_expname, + run_after=[prep_expname] if prep_submitted else run_after, + input_file=prepped_file, + output_dir=output_dir, + prompt_template=prompt_template, + gym_path=config["gym_path"], + gym_config_paths=config.get("gym_config_paths", []), + gym_agent_name=config["gym_agent_name"], + container=config.get("container", "nemo-rl"), + installation_command=config.get("installation_command"), + gym_uv_venv_dir=config.get("gym_uv_venv_dir", ""), + model_path=model_path, + num_gpus=num_gpus, + server_nodes=server_nodes, + num_chunks=config.get("num_chunks", 1), + num_random_seeds=config.get("num_random_seeds", 1), + inference_params=config.get("inference_params", {}), + vllm_extra=pv, + extra_record_fields=config.get("extra_record_fields"), + extra_record_field_mappers=config.get("extra_record_field_mappers"), + rerun_done=config.get("rerun_done", False), + ) + + # Genselect postprocess (select best answer -> output_file) + trim, run + # under the stage expname so downstream `run_after=[stage_expname]` waits. + trim_cmd = build_trim_cmd( + stage_name="gym_genselect_answers", + paths=[output_file], + domain_keep_fields=config.get("domain_keep_fields"), + ) + postprocess_cmd = ( + f"cp {output_dir}/output-rs0.jsonl {output_dir}/output.jsonl && " + "python -m nvflow.lib.sdg.document_grounded.genselect postprocess " + f"--input_dir={output_dir} " + f"--output_file={output_file} && " + f"{trim_cmd}" + ) + run_cmd( + ctx=wrap_arguments(postprocess_cmd), + cluster=cluster, + expname=expname, + log_dir=f"{output_dir}/postprocess-logs", + run_after=[gen_expname], + ) + + console.success(f"Genselect answer generation submitted -> {output_file}") + + def validate_config(self, config: dict[str, Any]) -> None: + """Validate required configuration fields.""" + for field in ( + "input_dir", + "output_file", + "prompt_template", + "gym_path", + "gym_agent_name", + ): + if not config.get(field): + raise ValueError(f"'{field}' is required in genselect_answers config") + if not config.get("policy_vllm") and not config.get("model"): + raise ValueError("Either 'policy_vllm.model_path' or 'model' is required") diff --git a/nvflow/lib/cli_cmd.py b/nvflow/lib/cli_cmd.py new file mode 100644 index 0000000..bbf098f --- /dev/null +++ b/nvflow/lib/cli_cmd.py @@ -0,0 +1,165 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Shared shell command builders for stage submission. + +Stages that submit ``python3 -m [positional ...] --flag ...`` +shell commands to nemo-skills' ``run_cmd`` / ``generate`` should use +:func:`build_python_cmd` to build the command string rather than +concatenating raw f-strings. ``shlex.quote`` ensures values containing +spaces, single quotes, or shell metacharacters do not break the rendered +command -- this matters because nemo-skills interpolates the command +into a Slurm shell wrapper at submission time. + +Usage:: + + from nvflow.lib.cli_cmd import build_python_cmd + + # Flag-only invocation: + rendered = build_python_cmd( + "nvflow.recipes.finance.utils.rl.regex_prefilter_questions", + input_file=Path("/lustre/foo/in.jsonl"), + output_kept=Path("/lustre/foo/kept.jsonl"), + ) + + # With positional args (e.g. for ``argparse`` scripts that take + # ``input_files`` positionally): + rendered = build_python_cmd( + "nvflow.recipes.finance.utils.shared.dataset_transformer", + Path("/lustre/sdg/final_result.jsonl"), + output_file="/lustre/out/final.jsonl", + num_chunks=10, + ) + # β†’ "python3 -m ...dataset_transformer " + # "/lustre/sdg/final_result.jsonl " + # "--output_file /lustre/out/final.jsonl --num_chunks 10" +""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +# All cluster containers in this repo provide ``python3`` (it is the +# canonical interpreter on every modern Linux base image we ship). We +# standardise on ``python3`` rather than ``python`` so ambiguity around +# the unversioned ``python`` symlink (absent in some minimal images) can +# never bite us. +_INTERPRETER = "python3" + + +def build_python_cmd( + module: str, + *positional: str | int | float | Path, + **flags: str | int | float | Path, +) -> str: + """Build a ``python3 -m [positional ...] --flag value ...`` shell command. + + Each positional and flag value is passed through :func:`shlex.quote` + so paths containing spaces, single quotes, or shell metacharacters + do not break the rendered command -- this command string is + interpolated by nemo-skills into a Slurm shell wrapper, so safe + quoting matters. + + Accepts ``str``, numeric types, or :class:`pathlib.Path` values; + non-string values are stringified via :func:`str` before quoting. + Positional args are emitted in argument order, then flags in + declaration order. This keeps the rendered command stable for + log-grepping and diffing across reruns. + + Args: + module: Fully-qualified Python module name (e.g. + ``"nvflow.recipes.finance.utils.shared.dataset_transformer"``). + *positional: Positional arguments emitted before any flags -- + useful for ``argparse``-style scripts that accept positional + inputs (e.g. one or more input file paths). + **flags: Keyword arguments rendered as ``-- `` + pairs in declaration order. Boolean flags (no value) must + be appended manually by the caller; this helper does not + support them because Python kwargs cannot express + "value-less" flags unambiguously. + + Returns: + A single-line shell command string suitable for nemo-skills' + ``run_cmd`` / ``generate`` ``ctx`` argument. + + Examples: + >>> build_python_cmd("foo.bar", input_file="/a/b.jsonl") + 'python3 -m foo.bar --input_file /a/b.jsonl' + >>> build_python_cmd("foo.bar", "/a/in.jsonl", output_file="/a/out.jsonl") + 'python3 -m foo.bar /a/in.jsonl --output_file /a/out.jsonl' + >>> build_python_cmd("foo.bar", input_file="/a path/with spaces.jsonl") + "python3 -m foo.bar --input_file '/a path/with spaces.jsonl'" + """ + parts = [_INTERPRETER, "-m", module] + for arg in positional: + parts.append(shlex.quote(str(arg))) + for flag, value in flags.items(): + parts.extend([f"--{flag}", shlex.quote(str(value))]) + return " ".join(parts) + + +def build_python_script_cmd( + script: str | Path, + *positional: str | int | float | Path, + **flags: str | int | float | Path, +) -> str: + """Build a ``python3 + + +""" + + +def build_index_html(summary: HopchainHtmlVisualizationSummary, title: str) -> str: + """Build the index page that links to all rendered HTML chunks.""" + if summary.page_files: + rows = "\n".join( + f""" + + {page.page_number} + {html.escape(page.file_name)} + {page.row_count} + {page.start_row}-{page.end_row} + + """ + for page in summary.page_files + ) + table_html = f""" + + + + + + + + + + + {rows} + +
PageFileRowsRange
+ """ + else: + table_html = '

No generated query rows were available to render.

' + + return f""" + + + + {html.escape(title)} + + + +

{html.escape(title)}

+
+
Total queries: {summary.total_queries}
+
Rendered queries: {summary.rendered_queries}
+
Rows per file: {summary.rows_per_file}
+
Pages generated: {len(summary.page_files)}
+
Queries input: {html.escape(summary.queries_input)}
+
Combinations input: {html.escape(summary.combinations_input)}
+
+ {table_html} + + +""" + + +def main() -> None: + """Entry point.""" + args = parse_args() + if args.sample_count is not None and args.sample_count <= 0: + raise SystemExit("error: --sample-count must be a positive integer") + queries_input = Path(args.queries_input) + combinations_input = Path(args.combinations_input) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + image_max_dimension = None if args.image_max_dimension == 0 else args.image_max_dimension + + queries = load_queries(queries_input) + combinations_by_id = load_combinations(combinations_input) + renderable_records = join_records(queries, combinations_by_id) + sample_seed = None + if args.sample_count is not None and args.sample_count < len(renderable_records): + random.Random(args.sample_seed).shuffle(renderable_records) + renderable_records = renderable_records[: args.sample_count] + sample_seed = args.sample_seed + + total_pages = ( + max(1, math.ceil(len(renderable_records) / args.rows_per_file)) if renderable_records else 0 + ) + page_summaries: list[HtmlPageSummary] = [] + + for page_number in range(1, total_pages + 1): + start_idx = (page_number - 1) * args.rows_per_file + end_idx = min(start_idx + args.rows_per_file, len(renderable_records)) + page_records = renderable_records[start_idx:end_idx] + page_file_name = f"hopchain_review_{page_number:04d}.html" + page_path = output_dir / page_file_name + page_path.write_text( + build_page_html( + page_records=page_records, + page_number=page_number, + total_pages=total_pages, + total_records=len(renderable_records), + rows_per_file=args.rows_per_file, + image_max_dimension=image_max_dimension, + title=args.title, + ) + ) + page_summaries.append( + HtmlPageSummary( + page_number=page_number, + file_name=page_file_name, + row_count=len(page_records), + start_row=start_idx + 1, + end_row=end_idx, + ) + ) + + summary = HopchainHtmlVisualizationSummary( + queries_input=str(queries_input), + combinations_input=str(combinations_input), + output_dir=str(output_dir), + index_file=str(output_dir / "index.html"), + total_queries=len(queries), + rendered_queries=len(renderable_records), + sample_seed=sample_seed, + rows_per_file=args.rows_per_file, + image_max_dimension=image_max_dimension, + page_files=page_summaries, + ) + + (output_dir / "index.html").write_text(build_index_html(summary, title=args.title)) + Path(args.summary).write_text(summary.model_dump_json(indent=2)) + logger.info("Wrote %s HTML page(s) to %s", len(page_summaries), output_dir) + + +if __name__ == "__main__": + main() diff --git a/nvflow/recipes/multimodal/utils/runtime_env.py b/nvflow/recipes/multimodal/utils/runtime_env.py new file mode 100644 index 0000000..10fafc0 --- /dev/null +++ b/nvflow/recipes/multimodal/utils/runtime_env.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Helpers for cluster-specific runtime settings.""" + +from __future__ import annotations + +from typing import Any + + +def resolve_partition( + config: dict[str, Any], + cluster: str, + *, + cpu: bool = False, + override_key: str = "partition", +) -> str: + """Resolve a stage partition from YAML or the selected cluster config.""" + if config.get(override_key): + return str(config[override_key]) + + from nemo_skills.pipeline.utils import get_cluster_config + + cluster_config = get_cluster_config( + cluster=cluster, + config_dir=config.get("cluster_config_dir"), + ) + key = "cpu_partition" if cpu else "partition" + partition = cluster_config.get(key) + if partition is None and cpu: + partition = cluster_config.get("partition") + if partition is None: + raise ValueError(f"Cluster config for {cluster!r} does not define {key!r}") + return str(partition) diff --git a/nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml b/nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml new file mode 100644 index 0000000..96973f7 --- /dev/null +++ b/nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter-demo.yaml @@ -0,0 +1,14 @@ +# Small, deterministic HopChain image-filter run used by the quick start. +# Put images under data/images, then run this file directly. + +_base_: hopchain-image-filter.yaml + +execution_id: demo + +stages: + image_filter: + image_directories: + - directory: ${project_root}/data/images + recursive: true + end_index: 100 + num_chunks: 1 diff --git a/nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml b/nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml new file mode 100644 index 0000000..eb94ee0 --- /dev/null +++ b/nvflow/recipes/multimodal/workflows/image_filter/hopchain-image-filter.yaml @@ -0,0 +1,69 @@ +# HopChain image filtering workflow +# This workflow is intentionally separate from the SDG workflow because it runs +# at a different scale and produces reusable filtered-image artifacts. + +recipe: multimodal + +workflow: + name: "hopchain_image_filter" + type: "image_filter" + description: "Filter candidate HopChain images before SDG synthesis" + +cluster: my_cluster + +model_profiles: + qwen: + model: qwen3.5-397b-a17b + server_type: sglang + server_gpus: 8 + server_nodes: 2 + server_args: >- + --model-path /hf_models/Qwen/Qwen3.5-397B-A17B + --served-model-name qwen3.5-397b-a17b + --context-length 131072 + --tp 16 + --ep-size 16 + --trust-remote-code + --reasoning-parser qwen3 + --mem-fraction-static 0.80 + --chunked-prefill-size 4096 + max_image_dimension: 2048 + tokens_to_generate: 16384 + max_concurrent_requests: 64 + time_min: 120 + use_base64_images: true + +# Run from the repository root. Machine-specific mounts, partitions, and +# container paths belong in cluster_configs/my_cluster.yaml. +project_root: ${oc.env:PWD} +base_data_dir: ${project_root}/outputs/hopchain +execution_id: full + +cluster_config_dir: ${project_root}/cluster_configs +image_filter_data_dir: ${base_data_dir}/image_filter + +directories: + image-filter: ${image_filter_data_dir}/execution/${execution_id}/image-filter + +pipeline_stages: + - image_filter + +stages: + image_filter: + image_directories: + - directory: ${project_root}/data/images + recursive: true + output_dir: ${directories.image-filter} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: nvflow/recipes/multimodal/prompts/hopchain_image_filter.txt + model_config: ${model_profiles.qwen} + min_complexity_score: 4 + allowed_quality_ratings: + - High + - Medium + temperature: 0.0 + top_p: 1.0 + tokens_to_generate: 16384 + num_chunks: 6 + dependencies: [] diff --git a/nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml b/nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml new file mode 100644 index 0000000..a2e2257 --- /dev/null +++ b/nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg-demo.yaml @@ -0,0 +1,40 @@ +# Small, deterministic HopChain SDG run used by the quick start. +# It consumes the output from hopchain-image-filter-demo.yaml and stops before +# the optional external API judge and downstream curation stages. + +_base_: hopchain-sdg.yaml + +execution_id: demo +image_filter_execution_id: demo +localize_dataloader_num_workers: 2 +localize_num_chunks: 1 +generate_multihop_queries_num_queries: 1 +generate_multihop_queries_num_chunks: 1 + +pipeline_stages: + - prepare_filtered_image_inputs + - preprocess_identify_categories + - identify_categories + - localize_instances + - sample_instance_combinations + - preprocess_generate_multihop_queries + - generate_multihop_queries + - verify_candidate_queries + - visualize_candidate_hopchain_data + +stages: + prepare_filtered_image_inputs: + sample_count: 25 + sample_seed: 42 + + identify_categories: + num_chunks: 1 + + localize_instances: + debug_save_annotated_images: false + + sample_instance_combinations: + max_combinations_per_image: 2 + + visualize_candidate_hopchain_data: + sample_count: 32 diff --git a/nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml b/nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml new file mode 100644 index 0000000..d5ae9ef --- /dev/null +++ b/nvflow/recipes/multimodal/workflows/sdg/hopchain-sdg.yaml @@ -0,0 +1,485 @@ +# HopChain SDG workflow + +recipe: multimodal + +workflow: + name: "hopchain_sdg" + type: "sdg" + description: "HopChain-inspired multimodal synthetic data generation workflow" + +cluster: my_cluster + +model_profiles: + qwen: + model: qwen3.5-397b-a17b + server_type: sglang + server_gpus: 8 + server_nodes: 2 + server_args: >- + --model-path /hf_models/Qwen/Qwen3.5-397B-A17B + --served-model-name qwen3.5-397b-a17b + --context-length 131072 + --tp 16 + --ep-size 16 + --trust-remote-code + --reasoning-parser qwen3 + --mem-fraction-static 0.80 + --chunked-prefill-size 4096 + max_image_dimension: 2048 + tokens_to_generate: 16384 + max_concurrent_requests: 64 + time_min: 120 + use_base64_images: true + + omni: + model: omni + server_type: vllm + server_gpus: 4 + server_nodes: 1 + server_args: >- + --model /hf_models/nvidia/omni-step70 + --served-model-name omni + --trust-remote-code + --max-model-len 32768 + --allowed-local-media-path / + --gpu-memory-utilization 0.9 + --limit-mm-per-prompt.image 1 + --mamba_ssm_cache_dtype float32 + --reasoning-parser nemotron_v3 + tokens_to_generate: 30000 + max_concurrent_requests: 4 + time_min: 240 + use_base64_images: false + + sam: + model: /hf_models/facebook/sam3.1/sam3.1_multiplex.pt + container: vllm + +# Run from the repository root. Machine-specific mounts, partitions, and +# container paths belong in cluster_configs/my_cluster.yaml. +project_root: ${oc.env:PWD} +base_data_dir: ${project_root}/outputs/hopchain +execution_id: full +image_filter_execution_id: full + +cluster_config_dir: ${project_root}/cluster_configs +sdg_data_dir: ${base_data_dir}/sdg +base_output_dir: ${sdg_data_dir}/execution/${execution_id} +source_kept_images_file: ${base_data_dir}/image_filter/execution/${image_filter_execution_id}/image-filter/kept_images.jsonl +gpu_time_min: 240 +cpu_time_min: 240 +max_image_dimension: 1536 +localize_dataloader_num_workers: 16 +localize_dataloader_prefetch_factor: 2 +localize_num_chunks: 6 +sample_instance_combinations_selection_strategy: balanced_by_category_then_area_confidence +sample_instance_combinations_min_instances: 3 +sample_instance_combinations_max_instances: 8 +sample_instance_combinations_max_instances_considered_per_image: 24 +sample_instance_combinations_max_instances_per_category: 2 +sample_instance_combinations_iou_dedup_threshold: 0.9 +sample_instance_combinations_iou_dedup_candidate_pool_size: 72 +sample_instance_combinations_debug_copy_selected_images: false +sample_instance_combinations_max_combinations_per_image: 6 +sample_instance_combinations_size_strategy: weighted_random +sample_instance_combinations_size_weights: + 3: 1 + 4: 2 + 5: 3 + 6: 4 + 7: 4 + 8: 4 +# area_confidence strategy params (used when selection_strategy is +# balanced_by_category_then_area_confidence or area_confidence) +sample_instance_combinations_area_confidence_area_weight: 0.5 +sample_instance_combinations_area_confidence_confidence_weight: 0.5 +sample_instance_combinations_sampling_seed: 20260430 +# sample_instance_combinations_min_confidence_threshold: 0.6 # uncomment to filter low-confidence instances +generate_multihop_queries_num_queries: 2 +generate_multihop_queries_target_hop_count_info: "4-10 hops" +generate_multihop_queries_temperature: 0.2 +generate_multihop_queries_top_p: 0.95 +generate_multihop_queries_tokens_to_generate: 32768 +# Heuristic: ~ (input_size * max_combinations_per_image / 2 * num_queries) / 1000 * factor. +# Check the actual step-3 combination count before enabling paid judge stages. +generate_multihop_queries_num_chunks: 18 # ~ 2000 * 6 / 2 * 2 / 1000 * 1.5 = 18 +verify_candidate_queries_min_hop_count: 4 +llm_judge_prompt_file: nvflow/recipes/multimodal/prompts/hopchain_llm_judge_answer_question.txt +llm_judge_temperature: 0.0 +llm_judge_top_p: 1.0 +llm_judge_reasoning_effort: medium +llm_judge_timeout_seconds: 300 +llm_judge_max_retries: 3 +llm_judge_max_workers: 64 +llm_judge_time_min: 240 +difficulty_filter_k: 5 +# 2K images * 6 max combos/image / 2 expected kept combos * 2 queries +# * ~70% LLM-judge accepted * difficulty_filter_k ~= 42K Omni requests. Omni is roughly +# 2K requests/hour/chunk, so use 21 chunks to target about 1 hour. +difficulty_filter_num_chunks: 21 +difficulty_filter_temperature: 0.6 +difficulty_filter_top_p: 0.95 +difficulty_filter_enable_thinking: false # note: omni-step70 always reasons via vLLM --reasoning-parser; this flag sets chat_template_kwargs per-datapoint which nemo_skills inference does not read +difficulty_filter_prompt_file: nvflow/recipes/multimodal/prompts/hopchain_llm_judge_answer_question.txt +difficulty_filter_min_pass_rate: 0.0 # inclusive lower bound; 0.0 = no lower filter +difficulty_filter_max_pass_rate: 1.0 # inclusive upper bound; 1.0 = no upper filter +sft_trace_k: 3 +sft_trace_generation_prompt_file: nvflow/recipes/multimodal/prompts/hopchain_sft_answer_question.txt +sft_trace_judge_prompt_file: nvflow/recipes/multimodal/prompts/hopchain_sft_trace_judge.txt +sft_trace_generation_num_chunks: 21 +sft_trace_judge_num_chunks: 21 +sft_trace_generation_temperature: 0.6 +sft_trace_generation_top_p: 0.95 +sft_trace_generation_tokens_to_generate: 32768 +sft_trace_judge_temperature: 0.6 +sft_trace_judge_top_p: 0.95 +sft_trace_judge_tokens_to_generate: 32768 + +directories: + step-0-prepare-filtered-inputs: ${base_output_dir}/step-0-prepare-filtered-inputs + step-1-identify-categories: ${base_output_dir}/step-1-identify-categories + step-1-preprocess-identify-categories: ${base_output_dir}/step-1-identify-categories/temp + step-2-localize-instances: ${base_output_dir}/step-2-localize-instances + step-3-sample-instance-combinations: ${base_output_dir}/step-3-sample-instance-combinations + step-4-preprocess-generate-multihop-queries: ${base_output_dir}/step-4-generate-multihop-queries/temp + step-4-generate-multihop-queries: ${base_output_dir}/step-4-generate-multihop-queries + step-5-verify-candidate-queries: ${base_output_dir}/step-5-verify-candidate-queries + step-6-visualize-candidate-hopchain-data: ${base_output_dir}/step-6-visualize-candidate-hopchain-data + step-7-judge-candidate-queries-openai: ${base_output_dir}/step-7-judge-candidate-queries-openai + step-8-reconcile-llm-judges: ${base_output_dir}/step-8-reconcile-llm-judges + step-9-visualize-reconciled-hopchain-data: ${base_output_dir}/step-9-visualize-reconciled-hopchain-data + step-10-preprocess-filter-easy-candidates: ${base_output_dir}/step-10-filter-easy-candidates/temp + step-10-filter-easy-candidates: ${base_output_dir}/step-10-filter-easy-candidates + step-11-preprocess-generate-sft-reasoning-traces: ${base_output_dir}/step-11-generate-sft-reasoning-traces/temp + step-11-generate-sft-reasoning-traces: ${base_output_dir}/step-11-generate-sft-reasoning-traces + step-12-preprocess-filter-sft-reasoning-traces: ${base_output_dir}/step-12-filter-sft-reasoning-traces/temp + step-12-filter-sft-reasoning-traces: ${base_output_dir}/step-12-filter-sft-reasoning-traces + +pipeline_stages: + - prepare_filtered_image_inputs + - preprocess_identify_categories + - identify_categories + - localize_instances + - sample_instance_combinations + - preprocess_generate_multihop_queries + - generate_multihop_queries + - verify_candidate_queries + - visualize_candidate_hopchain_data + - judge_candidate_queries_openai + - reconcile_llm_judges + - visualize_reconciled_hopchain_data + - preprocess_filter_easy_candidates + - filter_easy_candidates + - preprocess_generate_sft_reasoning_traces + - generate_sft_reasoning_traces + - preprocess_filter_sft_reasoning_traces + - filter_sft_reasoning_traces + +stages: + prepare_filtered_image_inputs: + run_name: ${execution_id} + input_file: ${source_kept_images_file} + output_file: ${directories.step-0-prepare-filtered-inputs}/filtered_image_inputs.jsonl + summary_file: ${directories.step-0-prepare-filtered-inputs}/summary.json + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + # sample_count: 100 # disabled for full-shard 10k execution; uncomment for small calibration runs + # sample_count_per_domain: 40 # disabled for full-shard 10k execution; uncomment for per-domain calibration runs + # sample_seed: 42 + time_min: ${cpu_time_min} + dependencies: [] + + preprocess_identify_categories: + run_name: ${execution_id} + input_file: ${directories.step-0-prepare-filtered-inputs}/filtered_image_inputs.jsonl + output_file: ${directories.step-1-preprocess-identify-categories}/input_openai_format.jsonl + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: nvflow/recipes/multimodal/prompts/hopchain_category_identification.txt + model_config: ${model_profiles.qwen} + max_image_dimension: ${max_image_dimension} + time_min: ${cpu_time_min} + dependencies: + - prepare_filtered_image_inputs + + identify_categories: + run_name: ${execution_id} + input_file: ${directories.step-0-prepare-filtered-inputs}/filtered_image_inputs.jsonl + preprocessed_input_file: ${directories.step-1-preprocess-identify-categories}/input_openai_format.jsonl + output_dir: ${directories.step-1-identify-categories} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: nvflow/recipes/multimodal/prompts/hopchain_category_identification.txt + model_config: ${model_profiles.qwen} + max_image_dimension: ${max_image_dimension} + temperature: 0.0 + top_p: 1.0 + tokens_to_generate: 8192 + num_chunks: 2 # ~ input_size / 1000 = 2000 / 1000 = 2 + time_min: ${gpu_time_min} + dependencies: + - preprocess_identify_categories + + localize_instances: + run_name: ${execution_id} + input_file: ${directories.step-1-identify-categories}/final_output.jsonl + output_dir: ${directories.step-2-localize-instances} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + container: ${model_profiles.sam.container} + prompt_file: nvflow/recipes/multimodal/prompts/hopchain_instance_localization_sam3.txt + model: ${model_profiles.sam.model} + filter_list: + - filter_method: min_image_size + min_w: 50 + min_h: 50 + max_image_dimension: ${max_image_dimension} + num_gpus: 1 + dataloader_num_workers: ${localize_dataloader_num_workers} + dataloader_prefetch_factor: ${localize_dataloader_prefetch_factor} + num_chunks: ${localize_num_chunks} + max_localization_phrases_per_category: 3 + prompt_alias_iou_dedup_threshold: 0.9 + debug_save_annotated_images: true + debug_annotated_images_dir: ${directories.step-2-localize-instances}/annotated_images + threshold: 0.5 + time_min: ${gpu_time_min} + dependencies: + - identify_categories + + sample_instance_combinations: + run_name: ${execution_id} + input_file: ${directories.step-2-localize-instances}/final_output.jsonl + output_file: ${directories.step-3-sample-instance-combinations}/instance_combinations.jsonl + summary_file: ${directories.step-3-sample-instance-combinations}/summary.json + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + min_instances: ${sample_instance_combinations_min_instances} + max_instances: ${sample_instance_combinations_max_instances} + selection_strategy: ${sample_instance_combinations_selection_strategy} + max_instances_considered_per_image: ${sample_instance_combinations_max_instances_considered_per_image} + max_instances_per_category: ${sample_instance_combinations_max_instances_per_category} + iou_dedup_threshold: ${sample_instance_combinations_iou_dedup_threshold} + iou_dedup_candidate_pool_size: ${sample_instance_combinations_iou_dedup_candidate_pool_size} + area_confidence_area_weight: ${sample_instance_combinations_area_confidence_area_weight} + area_confidence_confidence_weight: ${sample_instance_combinations_area_confidence_confidence_weight} + debug_copy_selected_images: ${sample_instance_combinations_debug_copy_selected_images} + debug_selected_images_dir: ${directories.step-3-sample-instance-combinations}/selected_images + max_combinations_per_image: ${sample_instance_combinations_max_combinations_per_image} + combination_size_strategy: ${sample_instance_combinations_size_strategy} + combination_size_weights: ${sample_instance_combinations_size_weights} + sampling_seed: ${sample_instance_combinations_sampling_seed} + time_min: ${cpu_time_min} + dependencies: + - localize_instances + + preprocess_generate_multihop_queries: + run_name: ${execution_id} + input_file: ${directories.step-3-sample-instance-combinations}/instance_combinations.jsonl + output_file: ${directories.step-4-preprocess-generate-multihop-queries}/input_openai_format.jsonl + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: nvflow/recipes/multimodal/prompts/hopchain_query_design.txt + model_config: ${model_profiles.qwen} + num_queries: ${generate_multihop_queries_num_queries} + target_hop_count_info: ${generate_multihop_queries_target_hop_count_info} + time_min: ${cpu_time_min} + dependencies: + - sample_instance_combinations + + generate_multihop_queries: + run_name: ${execution_id} + input_file: ${directories.step-3-sample-instance-combinations}/instance_combinations.jsonl + preprocessed_input_file: ${directories.step-4-preprocess-generate-multihop-queries}/input_openai_format.jsonl + output_dir: ${directories.step-4-generate-multihop-queries} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: nvflow/recipes/multimodal/prompts/hopchain_query_design.txt + model_config: ${model_profiles.qwen} + num_queries: ${generate_multihop_queries_num_queries} + target_hop_count_info: ${generate_multihop_queries_target_hop_count_info} + temperature: ${generate_multihop_queries_temperature} + top_p: ${generate_multihop_queries_top_p} + tokens_to_generate: ${generate_multihop_queries_tokens_to_generate} + num_chunks: ${generate_multihop_queries_num_chunks} # see top-level estimate + time_min: ${gpu_time_min} + dependencies: + - preprocess_generate_multihop_queries + + verify_candidate_queries: + run_name: ${execution_id} + input_file: ${directories.step-4-generate-multihop-queries}/final_output.jsonl + output_dir: ${directories.step-5-verify-candidate-queries} + output_file: ${directories.step-5-verify-candidate-queries}/verified_queries.jsonl + summary_file: ${directories.step-5-verify-candidate-queries}/summary.json + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + min_hop_count: ${verify_candidate_queries_min_hop_count} + time_min: ${cpu_time_min} + dependencies: + - generate_multihop_queries + + visualize_candidate_hopchain_data: + run_name: ${execution_id} + queries_input_file: ${directories.step-5-verify-candidate-queries}/final_candidates.jsonl + combinations_input_file: ${directories.step-3-sample-instance-combinations}/instance_combinations.jsonl + output_dir: ${directories.step-6-visualize-candidate-hopchain-data} + summary_file: ${directories.step-6-visualize-candidate-hopchain-data}/summary.json + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + rows_per_file: 100 + sample_count: 200 + sample_seed: 42 + image_max_dimension: 1024 + title: HopChain Candidate Query Review + time_min: ${cpu_time_min} + dependencies: + - verify_candidate_queries + + judge_candidate_queries_openai: + run_name: ${execution_id} + input_file: ${directories.step-5-verify-candidate-queries}/final_candidates.jsonl + output_file: ${directories.step-7-judge-candidate-queries-openai}/judged_candidates.jsonl + output_dir: ${directories.step-7-judge-candidate-queries-openai} + summary_file: ${directories.step-7-judge-candidate-queries-openai}/summary.json + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: ${llm_judge_prompt_file} + judge_name: openai_gpt_5_5 + provider: openai + model: gpt-5.5 + api_key_name: OPENAI_API_KEY + max_image_dimension: ${max_image_dimension} + temperature: ${llm_judge_temperature} + top_p: ${llm_judge_top_p} + reasoning_effort: ${llm_judge_reasoning_effort} + timeout_seconds: ${llm_judge_timeout_seconds} + max_retries: ${llm_judge_max_retries} + max_workers: ${llm_judge_max_workers} + time_min: ${llm_judge_time_min} + dependencies: + - verify_candidate_queries + + reconcile_llm_judges: + run_name: ${execution_id} + input_file: ${directories.step-5-verify-candidate-queries}/final_candidates.jsonl + output_file: ${directories.step-8-reconcile-llm-judges}/reconciled_queries.jsonl + output_dir: ${directories.step-8-reconcile-llm-judges} + summary_file: ${directories.step-8-reconcile-llm-judges}/summary.json + judge_output_files: + - ${directories.step-7-judge-candidate-queries-openai}/judged_candidates.jsonl + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + time_min: ${cpu_time_min} + dependencies: + - judge_candidate_queries_openai + + preprocess_filter_easy_candidates: + run_name: ${execution_id} + input_file: ${directories.step-8-reconcile-llm-judges}/reconciled_queries.jsonl + output_file: ${directories.step-10-preprocess-filter-easy-candidates}/input_openai_format.jsonl + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + prompt_file: ${difficulty_filter_prompt_file} + k: ${difficulty_filter_k} + time_min: ${cpu_time_min} + dependencies: + - reconcile_llm_judges + + filter_easy_candidates: + run_name: ${execution_id} + input_file: ${directories.step-8-reconcile-llm-judges}/reconciled_queries.jsonl + preprocessed_input_file: ${directories.step-10-preprocess-filter-easy-candidates}/input_openai_format.jsonl + output_dir: ${directories.step-10-filter-easy-candidates} + summary_file: ${directories.step-10-filter-easy-candidates}/summary.json + model_config: ${model_profiles.omni} + prompt_file: ${difficulty_filter_prompt_file} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + k: ${difficulty_filter_k} + num_chunks: ${difficulty_filter_num_chunks} + temperature: ${difficulty_filter_temperature} + top_p: ${difficulty_filter_top_p} + enable_thinking: ${difficulty_filter_enable_thinking} + min_pass_rate: ${difficulty_filter_min_pass_rate} + max_pass_rate: ${difficulty_filter_max_pass_rate} + time_min: ${gpu_time_min} + dependencies: + - preprocess_filter_easy_candidates + + preprocess_generate_sft_reasoning_traces: + run_name: ${execution_id} + input_file: ${directories.step-10-filter-easy-candidates}/kept_output.jsonl + output_file: ${directories.step-11-preprocess-generate-sft-reasoning-traces}/input_openai_format.jsonl + model_config: ${model_profiles.qwen} + prompt_file: ${sft_trace_generation_prompt_file} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + max_image_dimension: ${max_image_dimension} + k: ${sft_trace_k} + time_min: ${cpu_time_min} + dependencies: + - filter_easy_candidates + + generate_sft_reasoning_traces: + run_name: ${execution_id} + preprocessed_input_file: ${directories.step-11-preprocess-generate-sft-reasoning-traces}/input_openai_format.jsonl + output_file: ${directories.step-11-generate-sft-reasoning-traces}/final_result.jsonl + incorrect_output_file: ${directories.step-11-generate-sft-reasoning-traces}/incorrect_answers.jsonl + output_dir: ${directories.step-11-generate-sft-reasoning-traces} + summary_file: ${directories.step-11-generate-sft-reasoning-traces}/summary.json + model_config: ${model_profiles.qwen} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + num_chunks: ${sft_trace_generation_num_chunks} + temperature: ${sft_trace_generation_temperature} + top_p: ${sft_trace_generation_top_p} + tokens_to_generate: ${sft_trace_generation_tokens_to_generate} + time_min: ${gpu_time_min} + dependencies: + - preprocess_generate_sft_reasoning_traces + + preprocess_filter_sft_reasoning_traces: + run_name: ${execution_id} + input_file: ${directories.step-11-generate-sft-reasoning-traces}/final_result.jsonl + output_file: ${directories.step-12-preprocess-filter-sft-reasoning-traces}/input_openai_format.jsonl + prompt_file: ${sft_trace_judge_prompt_file} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + time_min: ${cpu_time_min} + dependencies: + - generate_sft_reasoning_traces + + filter_sft_reasoning_traces: + run_name: ${execution_id} + input_file: ${directories.step-11-generate-sft-reasoning-traces}/final_result.jsonl + preprocessed_input_file: ${directories.step-12-preprocess-filter-sft-reasoning-traces}/input_openai_format.jsonl + output_dir: ${directories.step-12-filter-sft-reasoning-traces} + summary_file: ${directories.step-12-filter-sft-reasoning-traces}/summary.json + model_config: ${model_profiles.qwen} + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + num_chunks: ${sft_trace_judge_num_chunks} + temperature: ${sft_trace_judge_temperature} + top_p: ${sft_trace_judge_top_p} + tokens_to_generate: ${sft_trace_judge_tokens_to_generate} + time_min: ${gpu_time_min} + dependencies: + - preprocess_filter_sft_reasoning_traces + + visualize_reconciled_hopchain_data: + run_name: ${execution_id} + queries_input_file: ${directories.step-8-reconcile-llm-judges}/final_candidates.jsonl + combinations_input_file: ${directories.step-3-sample-instance-combinations}/instance_combinations.jsonl + output_dir: ${directories.step-9-visualize-reconciled-hopchain-data} + summary_file: ${directories.step-9-visualize-reconciled-hopchain-data}/summary.json + project_root: ${project_root} + cluster_config_dir: ${cluster_config_dir} + rows_per_file: 100 + sample_count: 1000 + sample_seed: 42 + image_max_dimension: 1024 + title: HopChain LLM Judge Reconciliation Review + time_min: ${cpu_time_min} + dependencies: + - reconcile_llm_judges diff --git a/nvflow/utils/__init__.py b/nvflow/utils/__init__.py index 1cac4e7..f97e400 100644 --- a/nvflow/utils/__init__.py +++ b/nvflow/utils/__init__.py @@ -12,8 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Utility functions and helpers.""" +"""Utility functions and helpers. + +Only lightweight, dependency-free helpers are re-exported here. In +particular, the JSONL helpers in :mod:`nvflow.utils.jsonl` import +``orjson`` at module top-level and MUST be imported directly via +``from nvflow.utils.jsonl import ...``. Re-exporting them from this +package would force every consumer of :func:`setup_logger` (including +standalone workers like :mod:`nvflow.lib.rl.create_overlay`, which run +inside container images that do NOT ship ``orjson`` such as the vLLM +server container) to pay the ``orjson`` import cost -- and crash on +``ModuleNotFoundError`` when the dep is absent. + +If you need the JSONL helpers, import them explicitly:: + + from nvflow.utils.jsonl import iter_jsonl, write_jsonl, write_stats_json +""" from nvflow.utils.logging_setup import setup_logger -__all__ = ["setup_logger"] +__all__ = [ + "setup_logger", +] diff --git a/nvflow/utils/jsonl.py b/nvflow/utils/jsonl.py new file mode 100644 index 0000000..ae9dce3 --- /dev/null +++ b/nvflow/utils/jsonl.py @@ -0,0 +1,226 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Shared JSONL read / write helpers for the recipes/finance utilities. + +The same buffered JSONL pattern (orjson decode/encode, ``WRITE_BUFFER_SIZE``, +manual flush, error handling) appeared verbatim in eight files under +``recipes/finance/``. This module is the single source of truth so a fix +or change applies everywhere. + +Three primitives: + +- :func:`iter_jsonl` -- read a JSONL file lazily, with a choice of + malformed-line policy (``skip`` / ``raise`` / ``yield_error``). Empty + lines are always silently skipped. +- :class:`write_jsonl` -- buffered writer context manager. ``write()`` + accepts either a ``dict`` (orjson-encoded) or raw ``bytes`` (written + verbatim). The bytes pass-through is REQUIRED for the + validate_questions pure-row-filter contract: ``apply_validate_filter`` + passes raw SDG bytes through to preserve key ordering and float + formatting from the source file. +- :func:`write_stats_json` -- atomic write of a stats JSON via + ``tmp + os.replace()``. Crash-safe: a partial write leaves only + ``{path}.tmp``, never a truncated ``{path}``. + +Performance constraints honoured to keep adoption byte-identical to the +old in-line code: + +- Buffer flush uses ``b"\\n".join(buffer) + b"\\n"`` (one syscall per flush). +- Buffer threshold is ``>=`` (matches existing behaviour). +- Last flush appends a trailing newline (file always ends in ``\\n``). +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator +from pathlib import Path +from types import TracebackType +from typing import Any, Literal, overload + +import orjson + +DEFAULT_BUFFER_SIZE = 1000 +"""Default number of records to buffer before flushing to disk. + +Matches the historical ``WRITE_BUFFER_SIZE`` constant used in the +recipes/finance utilities. Kept centralised so future tuning needs to +happen in only one place. +""" + + +@overload +def iter_jsonl( + path: str | Path, + *, + on_error: Literal["skip", "raise"] = ..., +) -> Iterator[dict[str, Any]]: ... + + +@overload +def iter_jsonl( + path: str | Path, + *, + on_error: Literal["yield_error"], +) -> Iterator[tuple[dict[str, Any] | None, orjson.JSONDecodeError | None, bytes]]: ... + + +def iter_jsonl( + path: str | Path, + *, + on_error: Literal["skip", "raise", "yield_error"] = "skip", +) -> Iterator[Any]: + """Iterate JSONL records lazily. + + Empty lines are silently skipped in every mode (universal behaviour + today). Trailing whitespace is stripped before parsing. + + Args: + path: Path to the JSONL file (str or :class:`pathlib.Path`). + on_error: How to handle malformed JSON lines. + + - ``"skip"`` (default): silently skip the line. + - ``"raise"``: raise the underlying :class:`orjson.JSONDecodeError`. + - ``"yield_error"``: yield ``(None, exc, raw_line)`` for + malformed lines and ``(row, None, raw_line)`` for valid + ones. ``raw_line`` is the stripped source bytes -- callers + that emit audit records (e.g., + ``regex_prefilter_questions``) include a truncated decoded + copy in the dropped stream so an operator can inspect the + offending source line without re-opening the input file. + + Yields: + For ``skip`` / ``raise``: ``dict`` per valid line. + For ``yield_error``: + ``tuple[dict | None, orjson.JSONDecodeError | None, bytes]``. + """ + with open(path, "rb") as reader: + for raw in reader: + line = raw.strip() + if not line: + continue + try: + row = orjson.loads(line) + except orjson.JSONDecodeError as exc: + if on_error == "skip": + continue + if on_error == "raise": + raise + yield (None, exc, line) + continue + if on_error == "yield_error": + yield (row, None, line) + else: + yield row + + +class write_jsonl: # noqa: N801 -- callable-style API: pairs with iter_jsonl(path) function + """Buffered JSONL writer context manager. + + Named in lowercase intentionally so the call site reads as a + function-style helper paired with :func:`iter_jsonl`:: + + with write_jsonl(out_path) as out: + for row in iter_jsonl(in_path): + out.write(transform(row)) + + The lowercase naming violates :pep:`8` ``N801`` (CapWords for class + names); the rule is silenced via ``noqa`` because the readability + win at every call site outweighs the convention deviation, and the + pair ``iter_jsonl`` / ``write_jsonl`` is the established symmetry. + + Accepts ``dict`` (orjson-encoded with no options) or raw ``bytes`` + (written verbatim, used for byte-preserving pass-through). Bytes + must NOT contain a trailing newline -- the writer adds the line + terminator on flush, matching the historical + ``b"\\n".join(buffer) + b"\\n"`` pattern. + + Buffer flushes happen when ``len(buffer) >= buffer_size`` (matches + historical ``>=`` semantics) and once more on context exit if the + buffer is non-empty. The final flush appends a trailing newline so + the file ALWAYS ends in ``\\n`` -- matches historical behaviour and + ensures downstream tools that split on newlines see the last record. + """ + + def __init__(self, path: str | Path, *, buffer_size: int = DEFAULT_BUFFER_SIZE) -> None: + if buffer_size < 1: + raise ValueError(f"buffer_size must be >= 1 (got {buffer_size})") + self._path = path + self._buffer_size = buffer_size + self._buffer: list[bytes] = [] + self._fp: Any = None # opened in __enter__ + + def __enter__(self) -> write_jsonl: + # Open lazily on enter so the user can construct the writer + # outside a try/except without leaking file handles. + self._fp = open(self._path, "wb") + return self + + def write(self, row: dict[str, Any] | bytes) -> None: + """Append a record to the write buffer. + + ``dict`` rows are encoded via :func:`orjson.dumps` with no options + (no indenting, no key sort -- matches historical behaviour). + ``bytes`` rows are appended verbatim; they MUST be a single JSON + line WITHOUT a trailing newline (the writer adds line breaks on + flush via the join pattern). + """ + if isinstance(row, bytes): + encoded = row + else: + encoded = orjson.dumps(row) + self._buffer.append(encoded) + if len(self._buffer) >= self._buffer_size: + self._flush() + + def _flush(self) -> None: + if not self._buffer: + return + # Single syscall per flush -- matches historical performance. + self._fp.write(b"\n".join(self._buffer) + b"\n") + self._buffer.clear() + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + try: + self._flush() + finally: + self._fp.close() + self._fp = None + + +def write_stats_json(path: str | Path, fields: dict[str, Any]) -> None: + """Atomically write a stats JSON to ``path``. + + Encodes ``fields`` with :data:`orjson.OPT_INDENT_2` (matches the + historical pretty-printed stats files), writes to ``{path}.tmp``, + then renames to ``{path}`` via :func:`os.replace`. The tmp file + lives in the same directory as the target, so the rename is atomic + on POSIX (same-mount requirement satisfied). + + Crash semantics: a process killed mid-write leaves ``{path}.tmp`` + behind but never a truncated ``{path}``. Downstream tools that + cache ``{path}.exists()`` as "prior run completed" stay correct. + """ + target = Path(path) + tmp = target.with_name(target.name + ".tmp") + encoded = orjson.dumps(fields, option=orjson.OPT_INDENT_2) + with open(tmp, "wb") as f: + f.write(encoded) + os.replace(tmp, target) diff --git a/pyproject.toml b/pyproject.toml index eadd667..31761ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,16 +6,15 @@ authors = [ {name = "Your Team", email = "team@example.com"} ] readme = "README.md" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.14" license = {text = "Apache-2.0"} dependencies = [ - # NeMo-Skills for cluster job submission (brings ~200+ dependencies) - # This is the core framework we orchestrate - list it first for clarity - # Pinned to specific commit for reproducibility (nemo-skills doesn't use version tags) - # Pinned on: 2026-05-05 | Commit: 0229040 (consistent with nemo-skills:0229040 container) - # To update: Find commit with containers in cluster_configs/example-slurm.yaml - "nemo-skills @ git+https://github.com/NVIDIA/NeMo-Skills.git@022904023ad7a83a87662a313cf72e7df5891d55", + # Core framework we orchestrate (brings ~200+ deps). Pinned to a commit + # since nemo-skills has no version tags. + # Pinned 2026-08-02 | e06c9b90 (paired with nemo-rl v0.7.0; NVFlow v1.1.2). + # Must match NEMO_SKILLS_COMMIT in dockerfiles/Dockerfile.nemo-skills. + "nemo-skills @ git+https://github.com/NVIDIA/NeMo-Skills.git@e06c9b900177be3f60d6a3f99135bb5de9af9bed", # Configuration & Workflow "omegaconf>=2.3.0", # YAML config loading with variable interpolation @@ -27,10 +26,10 @@ dependencies = [ # Data handling "jsonlines>=4.0.0", # JSONL file reading/writing - # urllib3>=2.6.3 blocked by torchx<1.27 constraint (via nemo-skills β†’ nemo-run β†’ torchx) - - # Note: Heavy dependencies (torch, transformers, etc.) come from nemo-skills - # Actual compute happens on Slurm cluster inside containers + # Declared directly ONLY so the CPU-index source below can route it (keeps + # CUDA out of the x86_64 client). Floor set for CVE remediation; otherwise + # tracks nemo-skills' torch. + "torch>=2.13.0", ] [project.optional-dependencies] @@ -47,6 +46,7 @@ dev = [ "ruff>=0.1.0", "mypy>=1.7.0", "types-PyYAML", + "types-requests", ] [project.scripts] @@ -104,14 +104,49 @@ override-dependencies = [ # Mirrors the override added in nemo-skills/pyproject.toml (PR #1433), # but [tool.uv] from a dep is ignored β€” overrides must be at the top-level project. "httpx[http2]>=0.28.1", - "urllib3>=2.6.3", # Force minimum versions of transitive deps for security/maintenance updates - "cryptography>=47.0.0", - "Pillow>=12.2.0", + "cryptography>=48.0.1", + "Pillow>=12.3.0", "Pygments>=2.20.0", - "GitPython>=3.1.49", + "GitPython>=3.1.52", # 3.1.50 still resolves vulnerable; fixes land in 3.1.51/3.1.52 (Trivy HIGH) + # Nspec CVE remediation (2026-07-09) β€” long-term fixes + # ray[default]>=2.54.0 was previously overridden (removed in 4e1b7ec4 during rebase cleanup). + # Restoring at >=2.56.0 to cover both the functional fix (entrypoint_label_selector added + # in 2.54.0) and the Nspec CVE recommendation. Without this, nemo-run->torchx pins ray 2.53.0. + "ray[default]>=2.56.0", + "urllib3>=2.7.0", + "transformers>=5.13.0", + # CVE floors ported from !188 (litellm capped to 1.84.x to avoid the 1.91.x/aiohttp-4.0.0a1 conflict) + # lxml>=6.1.0 clears High CVE-2026-41066 (info disclosure / local file read) + "starlette>=1.3.1", + "litellm>=1.84.0,<1.85", + "lxml>=6.1.0", + "gradio>=6.20.0", +] + +# NSpect/Trivy HIGH CVE floors (2026-07-21) β€” transitive-only, raise lower bound. +constraint-dependencies = [ + "pyjwt>=2.12.0", + "python-multipart>=0.0.27", + "mcp>=1.28.1", + "msgpack>=1.2.1", + "nltk>=3.10.0", + "aiohttp>=3.13.3", + "pyarrow>=23.0.1", + "pyasn1>=0.6.4", # CVE-2026-59886: exact big-integer exponentiation DoS + "soupsieve>=2.8.4", ] +# x86_64 torch from CPU wheels (arm64 PyPI torch is already CPU-only), keeping +# CUDA runtime out of the amd64 client. explicit = only torch uses this index. +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[tool.uv.sources] +torch = [{ index = "pytorch-cpu", marker = "platform_machine == 'x86_64'" }] + [dependency-groups] dev = [ "pudb>=2025.1.3", diff --git a/scripts/_dump_rollout_fixtures.py b/scripts/_dump_rollout_fixtures.py new file mode 100644 index 0000000..cf5d818 --- /dev/null +++ b/scripts/_dump_rollout_fixtures.py @@ -0,0 +1,153 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""One-shot script to dump current renderer outputs into the test fixtures dir. + +Used to capture the initial baseline for the bash-renderer snapshot tests +in tests/test_rollout.py. Re-run after any *intentional* change to the +renderers to refresh fixtures, then audit the diff before committing. + +Usage: uv run python3 scripts/_dump_rollout_fixtures.py +""" + +from __future__ import annotations + +from pathlib import Path + +from nvflow.lib.rl.rollout import ( + _build_client_cmd, + _build_merge_cmd, + build_aggregate_cmd, + build_filter_cmd, +) + +FIXTURES = Path(__file__).parent.parent / "tests" / "fixtures" / "rollout" +FIXTURES.mkdir(parents=True, exist_ok=True) + + +def _client_cmd(**overrides: object) -> str: + """Render _build_client_cmd with a known-good baseline + overrides.""" + base: dict[str, object] = { + "output_dir": "/out/rollout", + "gym_path": "/opt/Gym", + "model_path": "/hf_models/Qwen/Qwen3-30B-A3B", + "agent_name": "finance_agent", + "input_data": "/data/train.jsonl", + "output_file": "/out/rollout/rs0/chunk_0.jsonl", + "done_file": "/out/rollout/rs0/chunk_0.jsonl.done", + "config_paths": "vllm.yaml,env.yaml,overlay.yaml", + "num_parallel": 512, + "job_label": "rs0_chunk0", + "policy_vllm_url": "http://policy:8000/v1", + "judge_vllm_url": "http://judge:8001/v1", + "judge_ng_run_overrides": ( + ' "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \\\n' + ' "+judge_model.responses_api_models.vllm_model.base_url=http://judge:8001/v1" \\\n' + ), + "max_num_samples": 0, + "chunk_id": 0, + "num_chunks": 8, + "responses_create_params": {"max_output_tokens": 32768, "temperature": 1.0}, + } + base.update(overrides) + return _build_client_cmd(**base) # type: ignore[arg-type] + + +# 1. Dual-server Qwen3-style: policy URL + judge URL + multi-chunk +(FIXTURES / "client_cmd_dual_server.txt").write_text(_client_cmd()) + +# 2. Policy-only (policy_as_judge or no judge): empty judge URL + overrides +(FIXTURES / "client_cmd_policy_only.txt").write_text( + _client_cmd(judge_vllm_url="", judge_ng_run_overrides="") +) + +# 3. Single-chunk path: chunk slicing branch must NOT emit +(FIXTURES / "client_cmd_no_chunk.txt").write_text(_client_cmd(num_chunks=1)) + +# 4. With max_num_samples cap (truncated input) +(FIXTURES / "client_cmd_max_samples.txt").write_text(_client_cmd(max_num_samples=10000)) + +# 5. Empty responses_create_params: no extra +responses_create_params.* lines +(FIXTURES / "client_cmd_no_rcp.txt").write_text(_client_cmd(responses_create_params={})) + + +# --- _build_merge_cmd --- +def _merge_cmd(**overrides: object) -> str: + base: dict[str, object] = { + "gym_path": "/opt/Gym", + "merged_file": "/out/rollout/output-rs0.jsonl", + "analysis_dir": "/out/rollout/analysis_rs0", + "seed_label": "rs0", + "num_chunks": 8, + "chunk_file_pattern": "/out/rollout/rs0/chunk_$i.jsonl", + "merged_done_file": "/out/rollout/output-rs0.jsonl.done", + "analyze_module": "nvflow.recipes.finance.utils.rl.analyze_rollouts", + "enrich_module": "nvflow.recipes.finance.utils.rl.enrich_rollouts", + "input_data": "/data/train.jsonl", + } + base.update(overrides) + return _build_merge_cmd(**base) # type: ignore[arg-type] + + +(FIXTURES / "merge_cmd_8chunks.txt").write_text(_merge_cmd()) +(FIXTURES / "merge_cmd_1chunk.txt").write_text(_merge_cmd(num_chunks=1)) + + +# --- build_aggregate_cmd --- +(FIXTURES / "aggregate_cmd_default.txt").write_text( + build_aggregate_cmd( + rollout_dir="/out/rollout", + aggregate_module="nvflow.recipes.finance.utils.rl.aggregate_seeds", + ) +) +(FIXTURES / "aggregate_cmd_custom_filename.txt").write_text( + build_aggregate_cmd( + rollout_dir="/out/rollout", + aggregate_module="nvflow.recipes.finance.utils.rl.aggregate_seeds", + difficulty_filename="custom_difficulty.jsonl", + ) +) + + +# --- build_filter_cmd --- +(FIXTURES / "filter_cmd_minimal.txt").write_text( + build_filter_cmd( + output_dir="/out", + difficulty_dir="/out/rollout", + filter_module="nvflow.recipes.finance.utils.rl.filter_training_data", + train_data="/in/train.jsonl", + validation_data="", + ) +) +(FIXTURES / "filter_cmd_full.txt").write_text( + build_filter_cmd( + output_dir="/out", + difficulty_dir="/out/rollout", + filter_module="nvflow.recipes.finance.utils.rl.filter_training_data", + train_data="/in/train.jsonl", + validation_data="/in/val.jsonl", + min_reward_std=1e-6, + policy_model="/hf_models/Qwen/Qwen3-30B-A3B", + judge_model="/hf_models/openai/gpt-oss-120b", + train_filename="train.jsonl", + val_filename="validation.jsonl", + difficulty_filename="difficulty.jsonl", + report_filename="filter_report.json", + ) +) + + +print("Wrote fixtures:") +for p in sorted(FIXTURES.iterdir()): + print(f" {p.name}: {p.stat().st_size} bytes") diff --git a/scripts/_dump_verify_fixtures.py b/scripts/_dump_verify_fixtures.py new file mode 100644 index 0000000..e4361ea --- /dev/null +++ b/scripts/_dump_verify_fixtures.py @@ -0,0 +1,109 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""One-shot script to dump current verify-renderer outputs into fixtures. + +Used to capture the baseline for the bash-renderer snapshot tests in +``tests/test_verify.py``. Re-run after any *intentional* change to +``_build_verify_cmd`` or ``_build_analysis_cmd``, then audit the diff +under ``tests/fixtures/verify/`` carefully before committing. + +Usage: uv run python3 scripts/_dump_verify_fixtures.py +""" + +from __future__ import annotations + +from pathlib import Path + +from nvflow.lib.rl.verify import _build_analysis_cmd, _build_verify_cmd + +FIXTURES = Path(__file__).parent.parent / "tests" / "fixtures" / "verify" +FIXTURES.mkdir(parents=True, exist_ok=True) + + +# Override blocks mirror what build_judge_ng_run_overrides() returns +# for each judge_mode -- keep them in sync so the snapshots reflect +# realistic call-site shapes rather than a stripped placeholder. + +_LOCAL_VLLM_OVERRIDES = ( + ' "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \\\n' + ' "+judge_model.responses_api_models.vllm_model.base_url=http://127.0.0.1:$JUDGE_PORT/v1" \\\n' + ' "+judge_model.responses_api_models.vllm_model.api_key=EMPTY" \\\n' + ' "+judge_model.responses_api_models.vllm_model.model=/hf_models/openai/gpt-oss-120b" \\\n' + ' "+judge_model.responses_api_models.vllm_model.return_token_id_information=false" \\\n' + ' "+judge_model.responses_api_models.vllm_model.uses_reasoning_parser=true" \\\n' + ' "+finance_env.resources_servers.finance_env.judge_model_server.name=judge_model" \\\n' +) + +_OPENAI_OVERRIDES = ( + ' "+judge_model.responses_api_models.openai_model.base_url=https://api.openai.com/v1" \\\n' + ' "+judge_model.responses_api_models.openai_model.api_key_env_var=OPENAI_API_KEY" \\\n' + ' "+judge_model.responses_api_models.openai_model.model=gpt-4o-mini" \\\n' + ' "+finance_env.resources_servers.finance_env.judge_model_server.name=judge_model" \\\n' +) + + +def _verify_cmd(**overrides: object) -> str: + """Render _build_verify_cmd with a known-good baseline + overrides.""" + base: dict[str, object] = { + "output_dir": "/out/verify", + "gym_path": "/opt/Gym", + "input_file": "/in/rollouts/output-rs0.jsonl", + "output_file": "/out/verify/rejudge/output-rs0.jsonl", + "done_file": "/out/verify/rejudge/output-rs0.jsonl.done", + "config_paths": "vllm.yaml,env.yaml,overlay.yaml", + "num_parallel": 8, + "job_label": "rejudge_rs0", + "judge_mode": "local_vllm", + "environment_name": "finance_env", + "judge_ng_run_overrides": _LOCAL_VLLM_OVERRIDES, + } + base.update(overrides) + return _build_verify_cmd(**base) # type: ignore[arg-type] + + +# 1. Local-vLLM judge: full overrides w/ uses_reasoning_parser +(FIXTURES / "verify_cmd_local_judge.txt").write_text(_verify_cmd()) + +# 2. OpenAI-API judge: shape of overrides differs (different keys) +(FIXTURES / "verify_cmd_openai_judge.txt").write_text( + _verify_cmd(judge_mode="openai", judge_ng_run_overrides=_OPENAI_OVERRIDES) +) + + +# --- _build_analysis_cmd --- +def _analysis_cmd(**overrides: object) -> str: + base: dict[str, object] = { + "rejudge_dir": "/out/verify/rejudge", + "gym_path": "/opt/Gym", + "analyze_module": "nvflow.recipes.finance.utils.rl.analyze_rollouts", + "analysis_entries": [ + ("rs0", "/out/verify/rejudge/output-rs0.jsonl"), + ("rs1", "/out/verify/rejudge/output-rs1.jsonl"), + ], + } + base.update(overrides) + return _build_analysis_cmd(**base) # type: ignore[arg-type] + + +(FIXTURES / "analysis_cmd_multi_seed.txt").write_text(_analysis_cmd()) +(FIXTURES / "analysis_cmd_single_seed.txt").write_text( + _analysis_cmd(analysis_entries=[("rs0", "/out/verify/rejudge/output-rs0.jsonl")]) +) +(FIXTURES / "analysis_cmd_empty_entries.txt").write_text(_analysis_cmd(analysis_entries=[])) + + +print("Wrote fixtures:") +for p in sorted(FIXTURES.iterdir()): + print(f" {p.name}: {p.stat().st_size} bytes") diff --git a/scripts/serve_vllm_patched.py b/scripts/serve_vllm_patched.py deleted file mode 100644 index b680e69..0000000 --- a/scripts/serve_vllm_patched.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Patched vLLM entrypoint for standalone ``vllm serve`` processes. - -Applies runtime workarounds **before** vLLM is imported, then builds and -runs the ``vllm.entrypoints.openai.api_server`` command directly. - -The upstream ``nemo_skills.inference.server.serve_vllm`` hardcodes -``tensor_parallel_size = num_gpus * num_nodes``, which assumes all GPUs -are for TP. This entrypoint replaces that with the formula:: - - tp = (num_gpus * num_nodes) // dp_size - -When ``--data-parallel-size`` is absent, ``dp_size`` defaults to 1 and the -formula gives the same result as the original code. This is model-agnostic -and backward-compatible. - -Tested with vLLM 0.17.1 and 0.18.1. -Other vLLM versions are safe -- patches skip gracefully if the expected -code snippets are not found, and vLLM starts normally unpatched. - -Active workarounds ------------------- -WORKAROUND(vllm-0.17-hermes) - Patches ``vllm/tool_parsers/hermes_tool_parser.py`` on disk so that - ``Hermes2ProToolParser.__init__`` caches tokenizer encode/decode results - behind a ``threading.Lock``. Fixes ``RuntimeError: Already borrowed`` - under concurrent chat-completion requests with tool calling enabled. - Remove when: vLLM ships fix from https://github.com/vllm-project/vllm/pull/35034 - -WORKAROUND(harmony-aarch64) - Pre-downloads tiktoken vocab files for gpt-oss models on aarch64. - The ``openai_harmony`` Rust binary cannot download them at runtime. - Remove when: openai_harmony ships a fixed aarch64 binary. - Tracking: https://github.com/openai/harmony/issues/71 -""" - -from __future__ import annotations - -import argparse -import os -import platform -import subprocess -import urllib.request -from importlib.util import find_spec -from pathlib import Path - -_TAG = "[serve_vllm_patched]" - -# --------------------------------------------------------------------------- -# WORKAROUND(vllm-0.17-hermes) -- hermes tool parser thread-safety -# -# Identical logic to RL/nemo_rl/models/generation/vllm/vllm_worker.py but -# applied on disk before vLLM is imported (standalone ``vllm serve`` has no -# in-process hook). -# -# Remove this entire section when vLLM ships the upstream fix. -# --------------------------------------------------------------------------- - -# --- Exact string snippets to locate and replace in hermes_tool_parser.py --- - -_OLD_IMPORT = "import json\nfrom collections.abc import Sequence" - -_NEW_IMPORT = "import json\nimport threading\nfrom collections.abc import Sequence" - -_OLD_CLASS_LINE = "class Hermes2ProToolParser(ToolParser):" - -_NEW_CLASS_LINE = ( - "class Hermes2ProToolParser(ToolParser):\n" - " _tokenizer_lock = threading.Lock()\n" - " _tokenizer_cache = {}" -) - -_OLD_INIT = ( - " self.tool_call_start_token_ids = self.model_tokenizer.encode(\n" - " self.tool_call_start_token, add_special_tokens=False\n" - " )\n" - " self.tool_call_end_token_ids = self.model_tokenizer.encode(\n" - " self.tool_call_end_token, add_special_tokens=False\n" - " )\n" - "\n" - " self.tool_call_start_token_array = [\n" - " self.model_tokenizer.decode([token_id])\n" - " for token_id in self.tool_call_start_token_ids\n" - " ]\n" - "\n" - " self.tool_call_end_token_array = [\n" - " self.model_tokenizer.decode([token_id])\n" - " for token_id in self.tool_call_end_token_ids\n" - " ]" -) - -_NEW_INIT = ( - " _tid = id(self.model_tokenizer)\n" - " if _tid in Hermes2ProToolParser._tokenizer_cache:\n" - " _cached = Hermes2ProToolParser._tokenizer_cache[_tid]\n" - " self.tool_call_start_token_ids = _cached['start_ids']\n" - " self.tool_call_end_token_ids = _cached['end_ids']\n" - " self.tool_call_start_token_array = _cached['start_array']\n" - " self.tool_call_end_token_array = _cached['end_array']\n" - " else:\n" - " with Hermes2ProToolParser._tokenizer_lock:\n" - " if _tid in Hermes2ProToolParser._tokenizer_cache:\n" - " _cached = Hermes2ProToolParser._tokenizer_cache[_tid]\n" - " self.tool_call_start_token_ids = _cached['start_ids']\n" - " self.tool_call_end_token_ids = _cached['end_ids']\n" - " self.tool_call_start_token_array = _cached['start_array']\n" - " self.tool_call_end_token_array = _cached['end_array']\n" - " else:\n" - " self.tool_call_start_token_ids = self.model_tokenizer.encode(\n" - " self.tool_call_start_token, add_special_tokens=False\n" - " )\n" - " self.tool_call_end_token_ids = self.model_tokenizer.encode(\n" - " self.tool_call_end_token, add_special_tokens=False\n" - " )\n" - " self.tool_call_start_token_array = [\n" - " self.model_tokenizer.decode([token_id])\n" - " for token_id in self.tool_call_start_token_ids\n" - " ]\n" - " self.tool_call_end_token_array = [\n" - " self.model_tokenizer.decode([token_id])\n" - " for token_id in self.tool_call_end_token_ids\n" - " ]\n" - " Hermes2ProToolParser._tokenizer_cache[_tid] = {\n" - " 'start_ids': self.tool_call_start_token_ids,\n" - " 'end_ids': self.tool_call_end_token_ids,\n" - " 'start_array': self.tool_call_start_token_array,\n" - " 'end_array': self.tool_call_end_token_array,\n" - " }" -) - - -def _patch_hermes_tool_parser() -> None: # WORKAROUND(vllm-0.17-hermes) - """Patch hermes_tool_parser.py on disk before vLLM imports it.""" - spec = find_spec("vllm") - if spec is None or not spec.submodule_search_locations: - print(f"{_TAG} vLLM not found -- skipping hermes patch.") - return - - base_dir = next(iter(spec.submodule_search_locations)) - target = os.path.join(base_dir, "tool_parsers", "hermes_tool_parser.py") - - if not os.path.exists(target): - print(f"{_TAG} {target} not found -- skipping hermes patch.") - return - - with open(target) as f: - content = f.read() - - if "_tokenizer_cache" in content: - print(f"{_TAG} Hermes patch already applied.") - return - - if _OLD_INIT not in content: - print(f"{_TAG} WARNING: Expected code snippet not found in {target}.") - print(f"{_TAG} The vLLM version may have changed -- skipping hermes patch.") - return - - content = content.replace(_OLD_IMPORT, _NEW_IMPORT, 1) - content = content.replace(_OLD_CLASS_LINE, _NEW_CLASS_LINE, 1) - content = content.replace(_OLD_INIT, _NEW_INIT, 1) - - with open(target, "w") as f: - f.write(content) - - print(f"{_TAG} Successfully patched {target} for thread-safety.") - - -# --------------------------------------------------------------------------- -# WORKAROUND(harmony-aarch64) -- tiktoken vocab download for gpt-oss on ARM -# -# Remove this entire section when openai_harmony ships a fixed aarch64 binary. -# --------------------------------------------------------------------------- - -_TIKTOKEN_FILES = { - "o200k_base.tiktoken": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken", - "cl100k_base.tiktoken": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", -} - - -def _ensure_tiktoken_cache() -> None: # WORKAROUND(harmony-aarch64) - """Download tiktoken vocab files if on aarch64 and set env vars. - - Skips the download when TIKTOKEN_CACHE_DIR or TIKTOKEN_RS_CACHE_DIR is - already set (e.g. pointing at files baked into the container), which is - required for airgap / offline environments. - """ - if platform.machine() not in ("aarch64", "arm64"): - return - - existing = os.environ.get("TIKTOKEN_CACHE_DIR") or os.environ.get("TIKTOKEN_RS_CACHE_DIR") - if existing: - print(f"{_TAG} Tiktoken cache already configured ({existing}), skipping download.") - os.environ.setdefault("TIKTOKEN_ENCODINGS_BASE", existing) - return - - cache_dir = Path("/tmp/tiktoken-encodings") - cache_dir.mkdir(parents=True, exist_ok=True) - - for filename, url in _TIKTOKEN_FILES.items(): - dest = cache_dir / filename - if dest.exists() and dest.stat().st_size > 0: - continue - try: - print(f"{_TAG} Downloading {filename} for aarch64 workaround...") - urllib.request.urlretrieve(url, dest) - print(f"{_TAG} {dest.stat().st_size:,} bytes -> {dest}") - except Exception as e: - print(f"{_TAG} WARNING: Failed to download {filename}: {e}") - return - - os.environ["TIKTOKEN_ENCODINGS_BASE"] = str(cache_dir) - os.environ.setdefault("TIKTOKEN_RS_CACHE_DIR", str(cache_dir)) - print(f"{_TAG} TIKTOKEN_ENCODINGS_BASE={cache_dir}") - - -# --------------------------------------------------------------------------- -# TP / DP arithmetic -# --------------------------------------------------------------------------- - - -def _extract_int_flag(args: list[str], flag: str, default: int = 1) -> int: - """Read an integer flag value from *args* without removing it. - - Supports both ``--flag N`` and ``--flag=N`` forms. - """ - for i, a in enumerate(args): - if a == flag and i + 1 < len(args): - return int(args[i + 1]) - if a.startswith(f"{flag}="): - return int(a.split("=", 1)[1]) - return default - - -def _has_flag(args: list[str], flag: str) -> bool: - """Return True if *flag* is already present in *args*.""" - return any(a == flag or a.startswith(f"{flag}=") for a in args) - - -# --------------------------------------------------------------------------- -# Entrypoint -# --------------------------------------------------------------------------- - - -def main(): - os.environ["TOKENIZERS_PARALLELISM"] = "false" - - _patch_hermes_tool_parser() - _ensure_tiktoken_cache() - - parser = argparse.ArgumentParser( - description="Patched vLLM server entrypoint with DP-aware TP calculation", - ) - parser.add_argument("--model", required=True, help="Model path or HF name") - parser.add_argument("--num_gpus", type=int, required=True) - parser.add_argument("--num_nodes", type=int, default=1) - parser.add_argument("--port", type=int, default=5000, help="Server port") - parser.add_argument("--no_verbose", action="store_true", help="Suppress request logs") - args, unknown = parser.parse_known_args() - - dp_size = _extract_int_flag(unknown, "--data-parallel-size") - total_gpus = args.num_gpus * args.num_nodes - tp_size = total_gpus // dp_size - - # Multi-node DP: vLLM must know how many DP ranks fit on the local - # (master) node, otherwise it tries to place all ranks locally. - dp_size_local = args.num_gpus // tp_size - if dp_size > 1 and not _has_flag(unknown, "--data-parallel-size-local"): - unknown.extend(["--data-parallel-size-local", str(dp_size_local)]) - - print(f"{_TAG} Deploying model {args.model}") - print( - f"{_TAG} GPUs: {total_gpus} total" - f" (num_gpus={args.num_gpus} x num_nodes={args.num_nodes})" - f" -> TP={tp_size}, DP={dp_size}, DP_local={dp_size_local}" - ) - - cmd_list = [ - "python3", - "-m", - "vllm.entrypoints.openai.api_server", - f"--model={args.model}", - f"--served-model-name={args.model}", - "--trust-remote-code", - "--host=0.0.0.0", - f"--port={args.port}", - f"--tensor-parallel-size={tp_size}", - ] - if args.no_verbose: - cmd_list.extend(["--disable-log-requests", "--disable-log-stats"]) - cmd_list.extend(unknown) - - print(f"{_TAG} Starting OpenAI Server") - print(f"{_TAG} cmd: {' '.join(cmd_list)}") - subprocess.run(cmd_list, check=True) - - -if __name__ == "__main__": - main() diff --git a/scripts/view_traces.py b/scripts/view_traces.py new file mode 100644 index 0000000..e022196 --- /dev/null +++ b/scripts/view_traces.py @@ -0,0 +1,615 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Lightweight NeMo-Gym rollout-trace viewer (stdlib only, no Gradio). + +Serves a tiny web UI to spot-check rollout traces ONE record at a time. The +server reads only the requested record (seek by line with a lazy offset cache), +so it handles multi-GB ``output-rs*.jsonl`` files without loading them. + +Design inspired by the (removed) ``nemo_gym/dataset_viewer.py`` but with zero +external dependencies -- pure Python stdlib. + +Usage: + uv run python scripts/view_traces.py [--root ] [--port 8800] + +``--root`` defaults to ``$NVFLOW_TRACE_ROOT`` if set, otherwise the current +directory. Then open the forwarded http://localhost: in your +browser. In Cursor / VS Code Remote the port is auto-forwarded over SSH -- just +click the "open in browser" notification (or use the Ports panel). + +Rendering: + * Rollout records (have ``responses_create_params`` + ``response``) are + rendered as a conversation: prompt -> reasoning -> tool calls -> tool + outputs -> final answer, plus a verdict header (reward / judge / expected). + * Any other record falls back to pretty-printed JSON. +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +MAX_CONTENT_CHARS = 20000 # per-turn display cap to keep the page responsive +MAX_FILES = 1000 +SCAN_MAXDEPTH = 8 +# Heavy / non-trace dirs to skip while scanning for *.jsonl (keeps the scan fast +# even when rooted at the whole grpo workflow dir -- e.g. the 14 GB SEC cache). +PRUNE_DIRS = { + ".venv", + "venv", + ".git", + "node_modules", + "__pycache__", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", + "cache", + "logs", + "filings", + "filings_metadata", +} +# jsonl filename substrings that are pipeline input artifacts (no traces) -- hidden +# from the dropdown to reduce noise. +SKIP_FILE_SUBSTRINGS = ("materialized_inputs", "chunk_input") + +# Point this at a single workflow output dir, not at the parent of the SEC dump +# tree -- scanning tens of thousands of filings makes the directory listing crawl. +DEFAULT_ROOT = os.environ.get("NVFLOW_TRACE_ROOT") or "." + + +# --------------------------------------------------------------------------- +# Record parsing (schema-aware, dict-based -- no openai/pydantic deps) +# --------------------------------------------------------------------------- +def _content_to_text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for c in content: + if isinstance(c, dict): + parts.append(c.get("text") or c.get("output") or json.dumps(c, indent=2)) + else: + parts.append(str(c)) + return "\n".join(parts) + if content is None: + return "" + return json.dumps(content, indent=2) + + +def _pretty_json_str(s): + try: + return json.dumps(json.loads(s), indent=2) + except Exception: + return s if isinstance(s, str) else json.dumps(s, indent=2) + + +def parse_item(m: dict): + """Convert one input/output item into {kind, title, content}.""" + if not m.get("type") and m.get("role"): + m = {**m, "type": "message"} + t = m.get("type") + if t == "message": + role = m.get("role", "assistant") + return {"kind": role, "title": "", "content": _content_to_text(m.get("content", ""))} + if t == "function_call": + name = m.get("name", "?") + return { + "kind": "tool_call", + "title": name, + "content": _pretty_json_str(m.get("arguments", "{}")), + } + if t == "function_call_output": + return { + "kind": "tool_output", + "title": "", + "content": _pretty_json_str(m.get("output", "")), + } + if t == "reasoning": + txt = "\n".join(s.get("text", "") for s in (m.get("summary") or []) if isinstance(s, dict)) + return { + "kind": "reasoning", + "title": "", + "content": txt or _content_to_text(m.get("content", "")), + } + return {"kind": "other", "title": t or "item", "content": json.dumps(m, indent=2)} + + +HEADER_FIELDS = ( + "question", + "problem", + "expected_answer", + "question_type", + "reward", + "judge_rating", + "judge_text", + "uuid", + "current_date", +) + + +def parse_record(rec: dict) -> dict: + """Return {schema, header, turns} for rollout records, else generic fallback.""" + header = {} + for k in HEADER_FIELDS: + if k in rec and rec[k] not in (None, ""): + header[k] = rec[k] + dp = rec.get("difficulty_profile") + if isinstance(dp, dict) and "avg_reward" in dp: + header["difficulty_profile.avg_reward"] = dp["avg_reward"] + + rcp = rec.get("responses_create_params") + resp = rec.get("response") + if isinstance(rcp, dict) and isinstance(resp, dict): + raw_inp = rcp.get("input") + if isinstance(raw_inp, str): + inp: list = [{"role": "user", "content": raw_inp}] + elif isinstance(raw_inp, list): + inp = raw_inp + else: + inp = [] + raw_out = resp.get("output") + out: list = raw_out if isinstance(raw_out, list) else [] + turns = [] + turn, step = 0, 0 + for m in inp + out: + if not isinstance(m, dict): + continue + if m.get("role") == "user": + turn += 1 + step = 0 + if m.get("type") == "function_call": + step += 1 + ti = parse_item(m) + content = ti["content"] or "" + if len(content) > MAX_CONTENT_CHARS: + ti["content"] = ( + content[:MAX_CONTENT_CHARS] + + f"\n\n... [truncated {len(content) - MAX_CONTENT_CHARS} chars]" + ) + ti["turn"], ti["step"] = turn, step + turns.append(ti) + return {"schema": "rollout", "header": header, "turns": turns} + + return {"schema": "generic", "header": header, "raw": rec} + + +# --------------------------------------------------------------------------- +# JSONL reader: read one record by index via a lazy byte-offset cache. +# --------------------------------------------------------------------------- +class JsonlReader: + """Reads a single record by index without loading the whole file. + + Maintains a lazily-grown byte-offset index so repeated/sequential access is + cheap. Thread-safe: a lock guards the shared offset cache (the HTTP server is + threaded). + """ + + def __init__(self, path: Path): + self.path = path + self.offsets: list[int] = [0] # offsets[i] = byte offset of line i + self.eof = False + self.count: int | None = None # filled lazily (Random / total) + self._lock = threading.Lock() + + def _extend_to(self, index: int): + """Grow the offset cache through line `index`. Caller must hold the lock.""" + if self.eof or index < len(self.offsets): + return + with open(self.path, "rb") as f: + f.seek(self.offsets[-1]) + i = len(self.offsets) - 1 + while i <= index: + line = f.readline() + if not line: + self.eof = True + self.count = len(self.offsets) - 1 + break + i += 1 + self.offsets.append(f.tell()) + + def get(self, index: int): + if index < 0: + return None + with self._lock: + self._extend_to(index) + if index >= len(self.offsets) - 1 and self.eof: + return None + offset = self.offsets[index] + # File I/O + parse outside the lock (independent of shared state). + try: + with open(self.path, "rb") as f: + f.seek(offset) + line = f.readline() + except OSError: + return None + if not line: + return None + try: + return json.loads(line) + except Exception: + return { + "_parse_error": True, + "raw_line": line.decode("utf-8", "replace")[:MAX_CONTENT_CHARS], + } + + def total(self) -> int: + """Record count via a single chunked byte scan (cached). Used by Random.""" + with self._lock: + if self.count is not None: + return self.count + n, last = 0, b"" + with open(self.path, "rb") as f: + while True: + chunk = f.read(1 << 20) + if not chunk: + break + n += chunk.count(b"\n") + last = chunk[-1:] + if last and last != b"\n": # final line without trailing newline + n += 1 + self.count = n + return n + + +# --------------------------------------------------------------------------- +# HTTP server +# --------------------------------------------------------------------------- +def list_jsonl_files(root: Path): + files = [] + root = root.resolve() + for dirpath, dirnames, filenames in os.walk(root): + rel = Path(dirpath).relative_to(root) + if len(rel.parts) > SCAN_MAXDEPTH: + dirnames[:] = [] + continue + dirnames[:] = [d for d in dirnames if d not in PRUNE_DIRS and not d.startswith(".")] + for fn in filenames: + if fn.endswith(".jsonl") and not any(s in fn for s in SKIP_FILE_SUBSTRINGS): + files.append(str((Path(dirpath) / fn).relative_to(root))) + if len(files) >= MAX_FILES: + return sorted(files) + return sorted(files) + + +def make_handler(root: Path): + root = root.resolve() + readers: dict[str, JsonlReader] = {} + readers_lock = threading.Lock() + + def reader_for(rel: str): + # Resolve safely within root (no path traversal). + target = (root / rel).resolve() + try: + target.relative_to(root) + except ValueError: + return None + if not target.is_file(): + return None + with readers_lock: + if rel not in readers: + readers[rel] = JsonlReader(target) + return readers[rel] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): # quiet + pass + + def _send_json(self, obj, code=200): + body = json.dumps(obj).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_html(self, text): + body = text.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): # noqa: N802 (BaseHTTPRequestHandler requires this name) + try: + self._route() + except Exception as e: # never leave the client hanging on a bare 500 + try: + self._send_json({"error": f"{type(e).__name__}: {e}"}, 500) + except Exception: + pass + + def _route(self): + parsed = urllib.parse.urlparse(self.path) + q = urllib.parse.parse_qs(parsed.query) + path = parsed.path + if path == "/" or path == "/index.html": + self._send_html(PAGE) + return + if path == "/api/files": + self._send_json({"root": str(root), "files": list_jsonl_files(root)}) + return + if path in ("/api/record", "/api/random"): + rel = (q.get("file") or [""])[0] + rd = reader_for(rel) + if rd is None: + self._send_json({"error": f"file not found under root: {rel}"}, 404) + return + if path == "/api/random": + total = rd.total() + index = random.randint(0, max(0, total - 1)) if total else 0 + else: + try: + index = int((q.get("index") or ["0"])[0]) + except ValueError: + index = 0 + rec = rd.get(index) + if rec is None: + self._send_json( + {"error": "no record at index", "index": index, "known_total": rd.count}, + 404, + ) + return + parsed_rec = parse_record(rec) + self._send_json( + {"index": index, "known_total": rd.count, "parsed": parsed_rec, "raw": rec} + ) + return + self._send_json({"error": "not found"}, 404) + + return Handler + + +# --------------------------------------------------------------------------- +# Frontend (single self-contained page) +# --------------------------------------------------------------------------- +PAGE = r""" +Rollout Trace Viewer + + +
+ + + + + + + + + + + +
+
+ + +""" + + +def main(): + ap = argparse.ArgumentParser( + description="Lightweight NeMo-Gym rollout-trace viewer (stdlib only)." + ) + ap.add_argument( + "--root", + default=DEFAULT_ROOT, + help=( + "Directory scanned for *.jsonl files (file dropdown). " + "Defaults to $NVFLOW_TRACE_ROOT, or the current directory." + ), + ) + ap.add_argument( + "--host", + default="127.0.0.1", + help="Bind host (default 127.0.0.1; use SSH/Cursor port-forward).", + ) + ap.add_argument("--port", type=int, default=8800, help="Bind port (default 8800).") + args = ap.parse_args() + + root = Path(args.root).expanduser() + if not root.is_dir(): + raise SystemExit(f"--root is not a directory: {root}") + root = root.resolve() + + handler = make_handler(root) + try: + httpd = ThreadingHTTPServer((args.host, args.port), handler) + except OSError as e: + raise SystemExit( + f"Could not bind {args.host}:{args.port} ({e}).\n" + "A viewer may already be running -- stop it, or pass a different --port." + ) from e + print(f"Rollout trace viewer serving {root}", flush=True) + print( + f" http://{args.host}:{args.port} (Cursor/VS Code will auto-forward this port)", + flush=True, + ) + print(" Ctrl+C to stop.", flush=True) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nstopping.") + httpd.shutdown() + + +if __name__ == "__main__": + main() diff --git a/scripts/wandb_consolidate.py b/scripts/wandb_consolidate.py index 3a1ae1e..259b5ff 100644 --- a/scripts/wandb_consolidate.py +++ b/scripts/wandb_consolidate.py @@ -13,37 +13,43 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Consolidate multiple WandB training runs into a single dashboard. +"""Consolidate multiple WandB training runs into one continuous dashboard. -Reads metrics from individual WandB runs (filtered by group) and writes -them to a single consolidated WandB run with continuous step numbers. +Each GRPO/SFT job (including each job of a resume chain) creates a separate WandB +run. This merges them post-hoc into a single run with continuous step numbers. -Each GRPO/SFT training job creates a separate WandB dashboard. This script -merges them post-hoc into one continuous view. +Source selection: + --run-ids ID... explicit runs (precise; best when many experiments share a group) + --group NAME all runs in a group, optionally narrowed by --since TIMESTAMP -Two modes: - fresh (default): Creates a new consolidated WandB run. - --append RUN_ID: Resumes an existing consolidated run and adds new data. - Only logs steps beyond what was previously consolidated. +Failure handling (automatic): runs are merged in chronological order and each is +truncated at its successor's resume step, so a failed/rolled-back job's stale tail +is dropped and the resumed job's data wins. Use --max-step to cap a trailing failed +run (the last run has no successor to bound it). + +Modes: + fresh (default) create a new consolidated run + --append RUN_ID extend an existing consolidated run with new steps only Usage: - # First time -- new consolidated dashboard from all runs in a group + # fresh, from explicit run-ids (--entity required if the project is owned by + # another team, e.g. nvidia; or set $WANDB_ENTITY) uv run python scripts/wandb_consolidate.py \\ - --project finance-grpo \\ - --group grpo-training-finance_sec_search \\ - --name gspo-qwen3-30b-consolidated + --project finance-grpo --entity nvidia \\ + --run-ids abc123 def456 ghi789 --name nano-consolidated - # After more training -- append new data to existing dashboard + # or by group + time window uv run python scripts/wandb_consolidate.py \\ - --project finance-grpo \\ - --group grpo-training-finance_sec_search \\ - --append + --project finance-grpo --group grpo-training-finance_sec_search \\ + --since 2026-06-28T19:00:00Z --name nano-consolidated - # Dry run -- show what would be logged + # append later as the chain grows uv run python scripts/wandb_consolidate.py \\ - --project finance-grpo \\ - --group grpo-training-finance_sec_search \\ - --dry-run + --project finance-grpo --run-ids abc123 def456 ghi789 jkl012 \\ + --append + + # preview only + ... --dry-run Requirements: pip install wandb @@ -69,8 +75,21 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--group", - required=True, - help="WandB group name to filter source runs (e.g., grpo-training-finance_sec_search).", + default=None, + help="WandB group to filter source runs. Required unless --run-ids is given.", + ) + parser.add_argument( + "--run-ids", + nargs="*", + default=None, + help="Explicit source run IDs to consolidate (precise selection; ignores " + "--group/--since). Preferred when multiple experiments share one group.", + ) + parser.add_argument( + "--since", + default=None, + help="Only include --group runs created at/after this ISO8601 UTC timestamp " + "(e.g. 2026-06-28T19:00:00Z).", ) parser.add_argument( "--name", @@ -79,11 +98,11 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--entity", - default=None, + default=os.environ.get("WANDB_ENTITY"), help=( - "WandB entity (team/user). If not set, uses the default from " - "'wandb login'. Required if your default entity differs from " - "the project owner." + "WandB entity (team/user). Defaults to $WANDB_ENTITY, else your " + "'wandb login' default. Required for --run-ids when the project is " + "owned by another entity (e.g. --entity nvidia)." ), ) parser.add_argument( @@ -103,40 +122,115 @@ def parse_args() -> argparse.Namespace: default=["ray/"], help="Skip metrics whose key starts with these prefixes (default: ray/).", ) + parser.add_argument( + "--skip-histograms", + action="store_true", + help="Do not carry over wandb.Histogram metrics (train/*/histogram, " + "validation/*/histogram). By default histograms ARE carried; only " + "artifact-backed tables/plots (e.g. full_result, *_plot_sample) are skipped.", + ) + parser.add_argument( + "--max-step", + type=int, + default=None, + help="Cap consolidation at this step (inclusive). Use to exclude a trailing " + "failed job's rolled-back steps beyond its last good checkpoint (the last " + "run has no successor to bound it automatically).", + ) parser.add_argument( "--output-dir", default=None, help=( - "Directory for WandB local run data. Defaults to " - "{training-logs}/wandb_consolidated/ if not set. " - "Prevents WandB from writing to the source tree." + "Directory for WandB local run data (default: " + "/outputs/wandb_consolidated/). Keeps WandB scratch files out " + "of the source tree." ), ) return parser.parse_args() def fetch_source_runs( - api, project: str, entity: str | None, group: str, exclude_id: str | None = None + api, + project: str, + entity: str | None, + group: str | None, + exclude_id: str | None = None, + run_ids: list[str] | None = None, + since: str | None = None, ): - """Fetch source runs in the given group, excluding the consolidated run.""" + """Fetch source runs to consolidate. + + Selection precedence: + - run_ids: fetch exactly those runs (group/since ignored). Use when several + experiments share one group. + - group: fetch the group, optionally filtered by `since` (created_at >= since). + Excludes the --append target and any prior consolidated runs (CONSOLIDATED_TAG). + + Runs are returned in CHRONOLOGICAL (created_at) order, not by max step, so that + on overlapping steps from a failed-then-resumed job the later run's data wins. + """ path = f"{entity}/{project}" if entity else project - runs = api.runs(path, filters={"group": group}) - if not runs: - print(f"No runs found in project={project}, group={group}") + + if run_ids: + candidates = [] + for rid in run_ids: + try: + candidates.append(api.run(f"{path}/{rid}")) + except Exception as e: # noqa: BLE001 + print( + f" WARN: could not fetch run id {rid}: {e}\n" + f" (looked under entity='{entity}'; pass --entity or set " + f"$WANDB_ENTITY if the project is owned by another entity, e.g. nvidia)" + ) + elif group: + candidates = list(api.runs(path, filters={"group": group})) + else: + print("Provide either --group or --run-ids.") sys.exit(1) run_list = [] - for r in runs: + for r in candidates: if exclude_id and r.id == exclude_id: continue if r.tags and CONSOLIDATED_TAG in r.tags: continue + if since and not run_ids and (getattr(r, "created_at", "") or "") < since: + continue run_list.append(r) - run_list.sort(key=lambda r: r.summary.get("_step", 0)) + if not run_list: + print( + f"No source runs matched (project={project}, group={group}, " + f"run_ids={run_ids}, since={since})." + ) + sys.exit(1) + + # Chronological: later (resume) runs supersede earlier failed ones on overlapping steps. + run_list.sort(key=lambda r: getattr(r, "created_at", "") or "") return run_list +def _to_wandb_histogram(wandb, raw: dict): + """Reconstruct a wandb.Histogram from a scan_history histogram dict. + + scan_history returns histograms as {"values": [counts...], + "packedBins": {"min": m, "size": s, "count": n}, "_type": "histogram"}. + wandb.Histogram(np_histogram=(counts, bin_edges)) needs len(bin_edges) == + len(counts) + 1, so we rebuild edges from the packed (min, size, count). + Returns None if the dict can't be reconstructed. + """ + try: + values = list(raw["values"]) + pb = raw["packedBins"] + mn, size, count = float(pb["min"]), float(pb["size"]), int(pb["count"]) + edges = [mn + i * size for i in range(count + 1)] + if len(edges) != len(values) + 1: + return None + return wandb.Histogram(np_histogram=(values, edges)) + except Exception: # noqa: BLE001 + return None + + def get_max_consolidated_step(api, project: str, entity: str | None, run_id: str) -> int: """Get the highest step already logged in the consolidated run.""" path = f"{entity}/{project}" if entity else project @@ -148,37 +242,92 @@ def get_max_consolidated_step(api, project: str, entity: str | None, run_id: str return -1 -def collect_metrics(runs, skip_prefixes: list[str], min_step: int = -1): - """Collect scalar metrics from source runs, skipping already-consolidated steps.""" - all_rows: list[tuple[int, dict[str, float]]] = [] - +def collect_metrics( + runs, + skip_prefixes: list[str], + min_step: int = -1, + max_step: int | None = None, + include_histograms: bool = True, +): + """Collect scalar (and optionally histogram) metrics from source runs. + + Runs are assumed in chronological order. + + Scalars are kept as-is. WandB histograms come back from scan_history as dicts + ({"_type": "histogram", "values": [...], "packedBins": {min,size,count}}); we + keep that raw dict and reconstruct a wandb.Histogram at log time. Artifact-backed + non-scalars (table-file sample dumps, *_plot_sample images) are always skipped. + + Resume-lineage truncation: each run is valid only up to where the *next* run + resumed (its successor's min step). Steps a job logged beyond its last + carried-forward checkpoint were rolled back, so they are dropped. This handles + failed jobs (empty valid range -> auto-dropped) and partial-success jobs + (keep the checkpointed prefix, drop the rolled-back tail) using only step data. + """ + # Pass 1: read each run's rows + its min step (runs are already chronological). + per_run: list[tuple[int | None, list[tuple[int, dict[str, object]]], object]] = [] for run in runs: print(f" Reading run: {run.name} ({run.id}), state={run.state}") - row_count = 0 - skipped = 0 + rows: list[tuple[int, dict[str, object]]] = [] for row in run.scan_history(): step = row.get("_step") if step is None: continue - if int(step) <= min_step: - skipped += 1 - continue - metrics = {} + step = int(step) + metrics: dict[str, object] = {} for k, v in row.items(): - if k.startswith("_"): - continue - if any(k.startswith(p) for p in skip_prefixes): + if k.startswith("_") or any(k.startswith(p) for p in skip_prefixes): continue if isinstance(v, int | float): metrics[k] = v + elif ( + include_histograms + and isinstance(v, dict) + and v.get("_type") == "histogram" + and v.get("values") is not None + and isinstance(v.get("packedBins"), dict) + ): + # Keep raw dict; reconstructed into wandb.Histogram at log time. + metrics[k] = v if metrics: - all_rows.append((int(step), metrics)) - row_count += 1 - msg = f" {row_count} steps collected" + rows.append((step, metrics)) + # Resume boundary = first *training* step (>0). step 0 is the per-job + # val_at_start artifact (re-logged every job), NOT the resume point, so it + # must be excluded or every run's boundary collapses to 0. + resume_step = min((s for s, _ in rows if s > 0), default=None) + per_run.append((resume_step, rows, run)) + + # Pass 2: bound each run at its successor's resume point + drop already-consolidated. + all_rows: list[tuple[int, dict[str, object]]] = [] + for i, (_, rows, run) in enumerate(per_run): + upper = None # exclusive upper bound = next run's min step + for j in range(i + 1, len(per_run)): + if per_run[j][0] is not None: + upper = per_run[j][0] + break + kept = truncated = skipped = capped = 0 + for step, metrics in rows: + if step <= min_step: + skipped += 1 + continue + if max_step is not None and step > max_step: + capped += 1 + continue + if upper is not None and step >= upper: + truncated += 1 + continue + all_rows.append((step, metrics)) + kept += 1 + msg = f" {run.name}: kept {kept} steps" + if truncated: + msg += f", dropped {truncated} rolled-back (>= successor resume @ {upper})" + if capped: + msg += f", dropped {capped} above --max-step {max_step}" if skipped: - msg += f" ({skipped} already consolidated, skipped)" + msg += f", skipped {skipped} already-consolidated" print(msg) + # Stable sort by step preserves chronological order on ties (later run wins). all_rows.sort(key=lambda x: x[0]) return all_rows @@ -190,14 +339,21 @@ def print_summary(rows: list[tuple[int, dict]], runs, min_step: int): return steps = [r[0] for r in rows] all_keys: set[str] = set() + hist_keys: set[str] = set() for _, metrics in rows: all_keys.update(metrics.keys()) + for k, v in metrics.items(): + if isinstance(v, dict) and v.get("_type") == "histogram": + hist_keys.add(k) print("\nConsolidation summary:") print(f" Source runs: {len(runs)}") if min_step >= 0: print(f" Already consolidated up to step: {min_step}") print(f" New steps: {len(steps)} (min={min(steps)}, max={max(steps)})") - print(f" Unique metrics: {len(all_keys)}") + print( + f" Unique metrics: {len(all_keys)} (scalars: {len(all_keys) - len(hist_keys)}, " + f"histograms: {len(hist_keys)})" + ) print(f" Total data points: {sum(len(m) for _, m in rows)}") @@ -215,12 +371,29 @@ def consolidate(args: argparse.Namespace): exclude_id = args.append print(f"Append mode: consolidated run {args.append}, max step = {min_step}") - print(f"Fetching source runs from project={args.project}, group={args.group}") - runs = fetch_source_runs(api, args.project, args.entity, args.group, exclude_id=exclude_id) + print( + f"Fetching source runs from project={args.project} " + f"(group={args.group}, run_ids={args.run_ids}, since={args.since})" + ) + runs = fetch_source_runs( + api, + args.project, + args.entity, + args.group, + exclude_id=exclude_id, + run_ids=args.run_ids, + since=args.since, + ) print(f"Found {len(runs)} source runs\n") print("Collecting metrics...") - rows = collect_metrics(runs, args.skip_prefixes or [], min_step=min_step) + rows = collect_metrics( + runs, + args.skip_prefixes or [], + min_step=min_step, + max_step=args.max_step, + include_histograms=not args.skip_histograms, + ) print_summary(rows, runs, min_step) if not rows: @@ -241,7 +414,7 @@ def consolidate(args: argparse.Namespace): print(f"\nAppending to existing WandB run: {args.append}") else: init_kwargs["name"] = args.name - init_kwargs["group"] = args.group + init_kwargs["group"] = args.group or args.name init_kwargs["tags"] = [CONSOLIDATED_TAG] print(f"\nCreating new WandB run: {args.name}") @@ -258,12 +431,25 @@ def consolidate(args: argparse.Namespace): print(f"URL: {run.url}") logged = 0 + hist_logged = 0 for step, metrics in rows: - run.log(metrics, step=step) - logged += len(metrics) + log_metrics = {} + for k, v in metrics.items(): + if isinstance(v, dict) and v.get("_type") == "histogram": + hv = _to_wandb_histogram(wandb, v) + if hv is not None: + log_metrics[k] = hv + hist_logged += 1 + else: + log_metrics[k] = v + run.log(log_metrics, step=step) + logged += len(log_metrics) run.finish() - print(f"\nDone. Logged {logged} data points across {len(rows)} steps.") + print( + f"\nDone. Logged {logged} data points across {len(rows)} steps " + f"({hist_logged} histogram points)." + ) print(f"Run ID: {run.id} (use with --append for future updates)") diff --git a/tests/fixtures/rollout/aggregate_cmd_custom_filename.txt b/tests/fixtures/rollout/aggregate_cmd_custom_filename.txt new file mode 100644 index 0000000..7fef1f1 --- /dev/null +++ b/tests/fixtures/rollout/aggregate_cmd_custom_filename.txt @@ -0,0 +1,7 @@ +set -e +echo "Cross-Seed Aggregation (pass@k)" +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.aggregate_seeds \ + "/out/rollout" \ + "/out/rollout/aggregate" \ + --output_filename "custom_difficulty.jsonl" +echo "Done. Results in /out/rollout/aggregate/" diff --git a/tests/fixtures/rollout/aggregate_cmd_default.txt b/tests/fixtures/rollout/aggregate_cmd_default.txt new file mode 100644 index 0000000..2e90380 --- /dev/null +++ b/tests/fixtures/rollout/aggregate_cmd_default.txt @@ -0,0 +1,7 @@ +set -e +echo "Cross-Seed Aggregation (pass@k)" +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.aggregate_seeds \ + "/out/rollout" \ + "/out/rollout/aggregate" \ + --output_filename "difficulty.jsonl" +echo "Done. Results in /out/rollout/aggregate/" diff --git a/tests/fixtures/rollout/client_cmd_dual_server.txt b/tests/fixtures/rollout/client_cmd_dual_server.txt new file mode 100644 index 0000000..9c54ac9 --- /dev/null +++ b/tests/fixtures/rollout/client_cmd_dual_server.txt @@ -0,0 +1,269 @@ +set -e + +OUTPUT_DIR=/out/rollout +GYM_PATH=/opt/Gym +UV_VENV_DIR=/opt/Gym +MODEL_PATH=/hf_models/Qwen/Qwen3-30B-A3B +AGENT_NAME=finance_agent +INPUT_DATA=/data/train.jsonl +OUTPUT_FILE=/out/rollout/rs0/chunk_0.jsonl +DONE_FILE=/out/rollout/rs0/chunk_0.jsonl.done +CONFIG_PATHS=vllm.yaml,env.yaml,overlay.yaml +NUM_PARALLEL=512 +JOB_LABEL=rs0_chunk0 +VLLM_URL="http://policy:8000/v1" +JUDGE_URL="http://judge:8001/v1" +CHUNK_ID=0 +NUM_CHUNKS=8 + +mkdir -p "$OUTPUT_DIR/logs" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +NG_RUN_PID="" + +cleanup() { + local _nvflow_exit=$? + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + # Suppress stderr via ``2>&-`` (close fd) instead of + # ``2>/dev/null`` so the cleanup trap stays quiet even if + # the container's /dev/null disappeared mid-script -- which + # is exactly what happened in the Nemotron-Nano smoke run + # where pyxis tore down the container while bash was still + # in the cleanup path, producing ``/dev/null: No such file + # or directory`` noise on top of the original failure. + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>&- && wait $NG_RUN_PID 2>&- || true + # Best-effort merge of .prev into -async. On success, finalize already + # merged and removed .prev so this block is a no-op. On failure/kill, + # this is a first attempt; the self-heal at next startup is the guarantee. + # Chain with && so .prev is NEVER deleted unless the merge succeeds. + if [ -n "$ASYNC_FILE" ] && [ -f "$ASYNC_FILE.prev" ] && [ "${PREV_MERGED:-0}" -eq 0 ]; then + echo "[Cleanup] Restoring previous results into -async for resume ..." + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.restored" \ + && { [ ! -f "$ASYNC_FILE" ] || cat "$ASYNC_FILE" >> "$ASYNC_FILE.restored"; } \ + && mv -f "$ASYNC_FILE.restored" "$ASYNC_FILE" \ + && rm -f "$ASYNC_FILE.prev" \ + || echo "[Cleanup] WARNING: merge failed β€” self-heal will recover on next start" + fi + if [ $_nvflow_exit -ne 0 ]; then + echo "[nvflow] Client exited with code $_nvflow_exit β€” cancelling job ${SLURM_JOB_ID}" + scancel "${SLURM_JOB_ID}" 2>&- || kill 0 2>&- || true + fi +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Rollout Collection [$JOB_LABEL]" +echo "============================================================" +echo "Model: $MODEL_PATH" +echo "Agent: $AGENT_NAME" +echo "Input data: $INPUT_DATA" +echo "Output file: $OUTPUT_FILE" +echo "Policy URL: $VLLM_URL" +[ -n "$JUDGE_URL" ] && echo "Judge URL: $JUDGE_URL" +echo "============================================================" +if [ -f "$DONE_FILE" ]; then + echo "Chunk already complete (.done exists) β€” skipping." + exit 0 +fi + +echo "" +echo "[Step 1/3] Waiting for vLLM servers ..." +wait_for_server "http://policy:8000/v1/models" "Policy vLLM" $$ 400 /dev/null +wait_for_server "http://judge:8001/v1/models" "Judge vLLM" $$ 400 /dev/null + +CHUNK_INPUT="" +if [ $NUM_CHUNKS -gt 1 ]; then + echo "" + echo "[Step 1a] Extracting chunk slice ..." + TOTAL_LINES=$(wc -l < "$INPUT_DATA") + EFFECTIVE=$TOTAL_LINES + MAX_SAMPLES=0 + if [ $MAX_SAMPLES -gt 0 ] && [ $MAX_SAMPLES -lt $TOTAL_LINES ]; then + EFFECTIVE=$MAX_SAMPLES + fi + CHUNK_SIZE=$(( (EFFECTIVE + NUM_CHUNKS - 1) / NUM_CHUNKS )) + START_LINE=$(( CHUNK_ID * CHUNK_SIZE + 1 )) + END_LINE=$(( (CHUNK_ID + 1) * CHUNK_SIZE )) + [ $END_LINE -gt $EFFECTIVE ] && END_LINE=$EFFECTIVE + CHUNK_INPUT="$OUTPUT_DIR/chunk_input_chunk$CHUNK_ID.jsonl" + head -n $END_LINE "$INPUT_DATA" | tail -n +$START_LINE > "$CHUNK_INPUT" + echo " Chunk $CHUNK_ID/$NUM_CHUNKS: lines $START_LINE-$END_LINE ($((END_LINE - START_LINE + 1)) samples)" + INPUT_DATA="$CHUNK_INPUT" +fi +ASYNC_FILE="$OUTPUT_FILE-async" +( + # Case A: output exists without .done β†’ restore to -async + if [ -f "$OUTPUT_FILE" ] && [ ! -f "$DONE_FILE" ]; then + echo "[Self-heal] Output file exists without .done β€” restoring to -async ..." + mv -f "$OUTPUT_FILE" "$ASYNC_FILE" + fi + + # Case B: orphaned .prev β†’ merge into -async + if [ -f "$ASYNC_FILE.prev" ]; then + echo "[Self-heal] Found orphaned .prev β€” merging into -async ..." + PREV_LINES=$(wc -l < "$ASYNC_FILE.prev") + ASYNC_LINES=0 + [ -f "$ASYNC_FILE" ] && ASYNC_LINES=$(wc -l < "$ASYNC_FILE") + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.healed" + [ -f "$ASYNC_FILE" ] && cat "$ASYNC_FILE" >> "$ASYNC_FILE.healed" + mv -f "$ASYNC_FILE.healed" "$ASYNC_FILE" && rm -f "$ASYNC_FILE.prev" + MERGED_LINES=$(wc -l < "$ASYNC_FILE") + echo " Recovered $PREV_LINES (prev) + $ASYNC_LINES (async) = $MERGED_LINES total rows" + fi + + # Case C: clean up orphaned temp files from prior crash + rm -f "$ASYNC_FILE.healed" "$ASYNC_FILE.restored" "$ASYNC_FILE.merged" +) || echo "[Self-heal] WARNING: recovery failed β€” continuing with available data" +REMAINING_INPUT="$OUTPUT_DIR/remaining_input_chunk$CHUNK_ID.jsonl" + +if ! PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.resume_filter "$ASYNC_FILE" "$INPUT_DATA" "$REMAINING_INPUT" 0; then + echo "ERROR: resume_filter failed" >&2 + exit 1 +fi + +if [ -f "$ASYNC_FILE" ] && [ ! -s "$REMAINING_INPUT" ]; then + echo "All rows already completed in -async -- finalizing." + cp -f "$ASYNC_FILE" "$OUTPUT_FILE" + touch "$DONE_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_FILE" "$ASYNC_FILE.prev" + echo "Done [$JOB_LABEL]." + exit 0 +fi + +HEAD_SERVER_PORT=$(find_free_port) + +cd "$GYM_PATH" + +echo "" +echo "[Step 2/3] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=$VLLM_URL" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=$MODEL_PATH" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+port_range_low=1024" \ + "+port_range_high=8999" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \ + "+judge_model.responses_api_models.vllm_model.base_url=http://judge:8001/v1" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 3/3] Collecting rollouts ..." +# Back up previous partial results before `gym eval run` clears the file. +ASYNC_BACKUP="" +PREV_MERGED=0 +if [ -s "$ASYNC_FILE" ]; then + ASYNC_BACKUP="$ASYNC_FILE.prev" + cp "$ASYNC_FILE" "$ASYNC_BACKUP" +fi +# Ensure clean slate for first attempt. Prior data is safe in .prev. +# Stale materialized_inputs from a prior Slurm job would cause +# resume_from_cache to load wrong task indexes. +rm -f "$ASYNC_FILE" +MATERIALIZED="$(dirname "$ASYNC_FILE")/$(basename "$ASYNC_FILE" .jsonl-async)_materialized_inputs.jsonl" +rm -f "$MATERIALIZED" +# Retry loop: the vLLM tokenizer race condition (RuntimeError: Already +# borrowed) can crash the client on the initial request burst. Retrying +# after a short delay shifts the timing and almost always succeeds. +# resume_from_cache=true ensures retries skip completed samples. +_NVFLOW_MAX_RETRIES=3 +_NVFLOW_RETRY_DELAY=15 +_NVFLOW_EXIT=0 +for _attempt in $(seq 1 $_NVFLOW_MAX_RETRIES); do + # Guard: truncate corrupted last line from SIGKILL mid-write + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + if [ "$(tail -c 1 "$ASYNC_FILE" | xxd -p)" != "0a" ]; then + head -n -1 "$ASYNC_FILE" > "$ASYNC_FILE.truncated" \ + && mv -f "$ASYNC_FILE.truncated" "$ASYNC_FILE" \ + || rm -f "$ASYNC_FILE.truncated" + echo "[nvflow] Truncated corrupted last line from $ASYNC_FILE" + fi + fi + set +e + gym eval run --no-serve \ + ${AGENT_NAME:++agent_name=$AGENT_NAME} \ + +input_jsonl_fpath=$REMAINING_INPUT \ + +output_jsonl_fpath=$ASYNC_FILE \ + +num_repeats=1 \ + +resume_from_cache=true \ + +num_samples_in_parallel=$NUM_PARALLEL \ + +head_server.host=127.0.0.1 \ + +head_server.port=$HEAD_SERVER_PORT \ + +responses_create_params.max_output_tokens=32768 \ + +responses_create_params.temperature=1.0 + _NVFLOW_EXIT=$? + set -e + [ $_NVFLOW_EXIT -eq 0 ] && break + # F6: `gym eval run` can exit non-zero AFTER all rollouts + # have already been written to ASYNC_FILE -- the gym's + # post-collection aggregate_metrics call returned 500 in the + # Nemotron-Nano smoke run, killing the process at 99% even + # though all 1000 rollouts were on disk. Retrying in that + # state is wasteful (re-runs everything) and dangerous if the + # container FS is being torn down (we hit + # ``/usr/bin/sleep: No such file or directory`` then). If + # ASYNC_FILE has at least as many rows as REMAINING_INPUT, we + # already have what we need; declare success and let finalize + # do its job. + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + _async_rows=$(wc -l < "$ASYNC_FILE" 2>&- || echo 0) + _input_rows=$(wc -l < "$REMAINING_INPUT" 2>&- || echo 0) + if [ "$_input_rows" -gt 0 ] && [ "$_async_rows" -ge "$_input_rows" ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT but $ASYNC_FILE has $_async_rows/$_input_rows rows -- treating as complete (post-collection error in gym, rollouts intact)." + _NVFLOW_EXIT=0 + break + fi + fi + if [ $_attempt -lt $_NVFLOW_MAX_RETRIES ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT (attempt $_attempt/$_NVFLOW_MAX_RETRIES). Retrying in ${_NVFLOW_RETRY_DELAY}s ..." + sleep $_NVFLOW_RETRY_DELAY + _NVFLOW_RETRY_DELAY=$((_NVFLOW_RETRY_DELAY * 2)) + fi +done +if [ $_NVFLOW_EXIT -ne 0 ]; then + echo "[nvflow] gym eval run failed after $_NVFLOW_MAX_RETRIES attempts." + exit $_NVFLOW_EXIT +fi + +# Merge previous partial results with new results. +if [ -n "$ASYNC_BACKUP" ] && [ -f "$ASYNC_BACKUP" ]; then + cat "$ASYNC_BACKUP" "$ASYNC_FILE" > "$ASYNC_FILE.merged" + mv -f "$ASYNC_FILE.merged" "$ASYNC_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_BACKUP" +fi +cp -f "$ASYNC_FILE" "$OUTPUT_FILE" +touch "$DONE_FILE" +# Safe to clean up β€” .done exists, chunk won't be rescheduled. +rm -f "$ASYNC_FILE" "$REMAINING_INPUT" +[ -n "$CHUNK_INPUT" ] && rm -f "$CHUNK_INPUT" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/fixtures/rollout/client_cmd_max_samples.txt b/tests/fixtures/rollout/client_cmd_max_samples.txt new file mode 100644 index 0000000..83c5902 --- /dev/null +++ b/tests/fixtures/rollout/client_cmd_max_samples.txt @@ -0,0 +1,269 @@ +set -e + +OUTPUT_DIR=/out/rollout +GYM_PATH=/opt/Gym +UV_VENV_DIR=/opt/Gym +MODEL_PATH=/hf_models/Qwen/Qwen3-30B-A3B +AGENT_NAME=finance_agent +INPUT_DATA=/data/train.jsonl +OUTPUT_FILE=/out/rollout/rs0/chunk_0.jsonl +DONE_FILE=/out/rollout/rs0/chunk_0.jsonl.done +CONFIG_PATHS=vllm.yaml,env.yaml,overlay.yaml +NUM_PARALLEL=512 +JOB_LABEL=rs0_chunk0 +VLLM_URL="http://policy:8000/v1" +JUDGE_URL="http://judge:8001/v1" +CHUNK_ID=0 +NUM_CHUNKS=8 + +mkdir -p "$OUTPUT_DIR/logs" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +NG_RUN_PID="" + +cleanup() { + local _nvflow_exit=$? + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + # Suppress stderr via ``2>&-`` (close fd) instead of + # ``2>/dev/null`` so the cleanup trap stays quiet even if + # the container's /dev/null disappeared mid-script -- which + # is exactly what happened in the Nemotron-Nano smoke run + # where pyxis tore down the container while bash was still + # in the cleanup path, producing ``/dev/null: No such file + # or directory`` noise on top of the original failure. + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>&- && wait $NG_RUN_PID 2>&- || true + # Best-effort merge of .prev into -async. On success, finalize already + # merged and removed .prev so this block is a no-op. On failure/kill, + # this is a first attempt; the self-heal at next startup is the guarantee. + # Chain with && so .prev is NEVER deleted unless the merge succeeds. + if [ -n "$ASYNC_FILE" ] && [ -f "$ASYNC_FILE.prev" ] && [ "${PREV_MERGED:-0}" -eq 0 ]; then + echo "[Cleanup] Restoring previous results into -async for resume ..." + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.restored" \ + && { [ ! -f "$ASYNC_FILE" ] || cat "$ASYNC_FILE" >> "$ASYNC_FILE.restored"; } \ + && mv -f "$ASYNC_FILE.restored" "$ASYNC_FILE" \ + && rm -f "$ASYNC_FILE.prev" \ + || echo "[Cleanup] WARNING: merge failed β€” self-heal will recover on next start" + fi + if [ $_nvflow_exit -ne 0 ]; then + echo "[nvflow] Client exited with code $_nvflow_exit β€” cancelling job ${SLURM_JOB_ID}" + scancel "${SLURM_JOB_ID}" 2>&- || kill 0 2>&- || true + fi +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Rollout Collection [$JOB_LABEL]" +echo "============================================================" +echo "Model: $MODEL_PATH" +echo "Agent: $AGENT_NAME" +echo "Input data: $INPUT_DATA" +echo "Output file: $OUTPUT_FILE" +echo "Policy URL: $VLLM_URL" +[ -n "$JUDGE_URL" ] && echo "Judge URL: $JUDGE_URL" +echo "============================================================" +if [ -f "$DONE_FILE" ]; then + echo "Chunk already complete (.done exists) β€” skipping." + exit 0 +fi + +echo "" +echo "[Step 1/3] Waiting for vLLM servers ..." +wait_for_server "http://policy:8000/v1/models" "Policy vLLM" $$ 400 /dev/null +wait_for_server "http://judge:8001/v1/models" "Judge vLLM" $$ 400 /dev/null + +CHUNK_INPUT="" +if [ $NUM_CHUNKS -gt 1 ]; then + echo "" + echo "[Step 1a] Extracting chunk slice ..." + TOTAL_LINES=$(wc -l < "$INPUT_DATA") + EFFECTIVE=$TOTAL_LINES + MAX_SAMPLES=10000 + if [ $MAX_SAMPLES -gt 0 ] && [ $MAX_SAMPLES -lt $TOTAL_LINES ]; then + EFFECTIVE=$MAX_SAMPLES + fi + CHUNK_SIZE=$(( (EFFECTIVE + NUM_CHUNKS - 1) / NUM_CHUNKS )) + START_LINE=$(( CHUNK_ID * CHUNK_SIZE + 1 )) + END_LINE=$(( (CHUNK_ID + 1) * CHUNK_SIZE )) + [ $END_LINE -gt $EFFECTIVE ] && END_LINE=$EFFECTIVE + CHUNK_INPUT="$OUTPUT_DIR/chunk_input_chunk$CHUNK_ID.jsonl" + head -n $END_LINE "$INPUT_DATA" | tail -n +$START_LINE > "$CHUNK_INPUT" + echo " Chunk $CHUNK_ID/$NUM_CHUNKS: lines $START_LINE-$END_LINE ($((END_LINE - START_LINE + 1)) samples)" + INPUT_DATA="$CHUNK_INPUT" +fi +ASYNC_FILE="$OUTPUT_FILE-async" +( + # Case A: output exists without .done β†’ restore to -async + if [ -f "$OUTPUT_FILE" ] && [ ! -f "$DONE_FILE" ]; then + echo "[Self-heal] Output file exists without .done β€” restoring to -async ..." + mv -f "$OUTPUT_FILE" "$ASYNC_FILE" + fi + + # Case B: orphaned .prev β†’ merge into -async + if [ -f "$ASYNC_FILE.prev" ]; then + echo "[Self-heal] Found orphaned .prev β€” merging into -async ..." + PREV_LINES=$(wc -l < "$ASYNC_FILE.prev") + ASYNC_LINES=0 + [ -f "$ASYNC_FILE" ] && ASYNC_LINES=$(wc -l < "$ASYNC_FILE") + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.healed" + [ -f "$ASYNC_FILE" ] && cat "$ASYNC_FILE" >> "$ASYNC_FILE.healed" + mv -f "$ASYNC_FILE.healed" "$ASYNC_FILE" && rm -f "$ASYNC_FILE.prev" + MERGED_LINES=$(wc -l < "$ASYNC_FILE") + echo " Recovered $PREV_LINES (prev) + $ASYNC_LINES (async) = $MERGED_LINES total rows" + fi + + # Case C: clean up orphaned temp files from prior crash + rm -f "$ASYNC_FILE.healed" "$ASYNC_FILE.restored" "$ASYNC_FILE.merged" +) || echo "[Self-heal] WARNING: recovery failed β€” continuing with available data" +REMAINING_INPUT="$OUTPUT_DIR/remaining_input_chunk$CHUNK_ID.jsonl" + +if ! PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.resume_filter "$ASYNC_FILE" "$INPUT_DATA" "$REMAINING_INPUT" 0; then + echo "ERROR: resume_filter failed" >&2 + exit 1 +fi + +if [ -f "$ASYNC_FILE" ] && [ ! -s "$REMAINING_INPUT" ]; then + echo "All rows already completed in -async -- finalizing." + cp -f "$ASYNC_FILE" "$OUTPUT_FILE" + touch "$DONE_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_FILE" "$ASYNC_FILE.prev" + echo "Done [$JOB_LABEL]." + exit 0 +fi + +HEAD_SERVER_PORT=$(find_free_port) + +cd "$GYM_PATH" + +echo "" +echo "[Step 2/3] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=$VLLM_URL" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=$MODEL_PATH" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+port_range_low=1024" \ + "+port_range_high=8999" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \ + "+judge_model.responses_api_models.vllm_model.base_url=http://judge:8001/v1" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 3/3] Collecting rollouts ..." +# Back up previous partial results before `gym eval run` clears the file. +ASYNC_BACKUP="" +PREV_MERGED=0 +if [ -s "$ASYNC_FILE" ]; then + ASYNC_BACKUP="$ASYNC_FILE.prev" + cp "$ASYNC_FILE" "$ASYNC_BACKUP" +fi +# Ensure clean slate for first attempt. Prior data is safe in .prev. +# Stale materialized_inputs from a prior Slurm job would cause +# resume_from_cache to load wrong task indexes. +rm -f "$ASYNC_FILE" +MATERIALIZED="$(dirname "$ASYNC_FILE")/$(basename "$ASYNC_FILE" .jsonl-async)_materialized_inputs.jsonl" +rm -f "$MATERIALIZED" +# Retry loop: the vLLM tokenizer race condition (RuntimeError: Already +# borrowed) can crash the client on the initial request burst. Retrying +# after a short delay shifts the timing and almost always succeeds. +# resume_from_cache=true ensures retries skip completed samples. +_NVFLOW_MAX_RETRIES=3 +_NVFLOW_RETRY_DELAY=15 +_NVFLOW_EXIT=0 +for _attempt in $(seq 1 $_NVFLOW_MAX_RETRIES); do + # Guard: truncate corrupted last line from SIGKILL mid-write + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + if [ "$(tail -c 1 "$ASYNC_FILE" | xxd -p)" != "0a" ]; then + head -n -1 "$ASYNC_FILE" > "$ASYNC_FILE.truncated" \ + && mv -f "$ASYNC_FILE.truncated" "$ASYNC_FILE" \ + || rm -f "$ASYNC_FILE.truncated" + echo "[nvflow] Truncated corrupted last line from $ASYNC_FILE" + fi + fi + set +e + gym eval run --no-serve \ + ${AGENT_NAME:++agent_name=$AGENT_NAME} \ + +input_jsonl_fpath=$REMAINING_INPUT \ + +output_jsonl_fpath=$ASYNC_FILE \ + +num_repeats=1 \ + +resume_from_cache=true \ + +num_samples_in_parallel=$NUM_PARALLEL \ + +head_server.host=127.0.0.1 \ + +head_server.port=$HEAD_SERVER_PORT \ + +responses_create_params.max_output_tokens=32768 \ + +responses_create_params.temperature=1.0 + _NVFLOW_EXIT=$? + set -e + [ $_NVFLOW_EXIT -eq 0 ] && break + # F6: `gym eval run` can exit non-zero AFTER all rollouts + # have already been written to ASYNC_FILE -- the gym's + # post-collection aggregate_metrics call returned 500 in the + # Nemotron-Nano smoke run, killing the process at 99% even + # though all 1000 rollouts were on disk. Retrying in that + # state is wasteful (re-runs everything) and dangerous if the + # container FS is being torn down (we hit + # ``/usr/bin/sleep: No such file or directory`` then). If + # ASYNC_FILE has at least as many rows as REMAINING_INPUT, we + # already have what we need; declare success and let finalize + # do its job. + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + _async_rows=$(wc -l < "$ASYNC_FILE" 2>&- || echo 0) + _input_rows=$(wc -l < "$REMAINING_INPUT" 2>&- || echo 0) + if [ "$_input_rows" -gt 0 ] && [ "$_async_rows" -ge "$_input_rows" ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT but $ASYNC_FILE has $_async_rows/$_input_rows rows -- treating as complete (post-collection error in gym, rollouts intact)." + _NVFLOW_EXIT=0 + break + fi + fi + if [ $_attempt -lt $_NVFLOW_MAX_RETRIES ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT (attempt $_attempt/$_NVFLOW_MAX_RETRIES). Retrying in ${_NVFLOW_RETRY_DELAY}s ..." + sleep $_NVFLOW_RETRY_DELAY + _NVFLOW_RETRY_DELAY=$((_NVFLOW_RETRY_DELAY * 2)) + fi +done +if [ $_NVFLOW_EXIT -ne 0 ]; then + echo "[nvflow] gym eval run failed after $_NVFLOW_MAX_RETRIES attempts." + exit $_NVFLOW_EXIT +fi + +# Merge previous partial results with new results. +if [ -n "$ASYNC_BACKUP" ] && [ -f "$ASYNC_BACKUP" ]; then + cat "$ASYNC_BACKUP" "$ASYNC_FILE" > "$ASYNC_FILE.merged" + mv -f "$ASYNC_FILE.merged" "$ASYNC_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_BACKUP" +fi +cp -f "$ASYNC_FILE" "$OUTPUT_FILE" +touch "$DONE_FILE" +# Safe to clean up β€” .done exists, chunk won't be rescheduled. +rm -f "$ASYNC_FILE" "$REMAINING_INPUT" +[ -n "$CHUNK_INPUT" ] && rm -f "$CHUNK_INPUT" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/fixtures/rollout/client_cmd_no_chunk.txt b/tests/fixtures/rollout/client_cmd_no_chunk.txt new file mode 100644 index 0000000..4a163bc --- /dev/null +++ b/tests/fixtures/rollout/client_cmd_no_chunk.txt @@ -0,0 +1,269 @@ +set -e + +OUTPUT_DIR=/out/rollout +GYM_PATH=/opt/Gym +UV_VENV_DIR=/opt/Gym +MODEL_PATH=/hf_models/Qwen/Qwen3-30B-A3B +AGENT_NAME=finance_agent +INPUT_DATA=/data/train.jsonl +OUTPUT_FILE=/out/rollout/rs0/chunk_0.jsonl +DONE_FILE=/out/rollout/rs0/chunk_0.jsonl.done +CONFIG_PATHS=vllm.yaml,env.yaml,overlay.yaml +NUM_PARALLEL=512 +JOB_LABEL=rs0_chunk0 +VLLM_URL="http://policy:8000/v1" +JUDGE_URL="http://judge:8001/v1" +CHUNK_ID=0 +NUM_CHUNKS=1 + +mkdir -p "$OUTPUT_DIR/logs" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +NG_RUN_PID="" + +cleanup() { + local _nvflow_exit=$? + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + # Suppress stderr via ``2>&-`` (close fd) instead of + # ``2>/dev/null`` so the cleanup trap stays quiet even if + # the container's /dev/null disappeared mid-script -- which + # is exactly what happened in the Nemotron-Nano smoke run + # where pyxis tore down the container while bash was still + # in the cleanup path, producing ``/dev/null: No such file + # or directory`` noise on top of the original failure. + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>&- && wait $NG_RUN_PID 2>&- || true + # Best-effort merge of .prev into -async. On success, finalize already + # merged and removed .prev so this block is a no-op. On failure/kill, + # this is a first attempt; the self-heal at next startup is the guarantee. + # Chain with && so .prev is NEVER deleted unless the merge succeeds. + if [ -n "$ASYNC_FILE" ] && [ -f "$ASYNC_FILE.prev" ] && [ "${PREV_MERGED:-0}" -eq 0 ]; then + echo "[Cleanup] Restoring previous results into -async for resume ..." + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.restored" \ + && { [ ! -f "$ASYNC_FILE" ] || cat "$ASYNC_FILE" >> "$ASYNC_FILE.restored"; } \ + && mv -f "$ASYNC_FILE.restored" "$ASYNC_FILE" \ + && rm -f "$ASYNC_FILE.prev" \ + || echo "[Cleanup] WARNING: merge failed β€” self-heal will recover on next start" + fi + if [ $_nvflow_exit -ne 0 ]; then + echo "[nvflow] Client exited with code $_nvflow_exit β€” cancelling job ${SLURM_JOB_ID}" + scancel "${SLURM_JOB_ID}" 2>&- || kill 0 2>&- || true + fi +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Rollout Collection [$JOB_LABEL]" +echo "============================================================" +echo "Model: $MODEL_PATH" +echo "Agent: $AGENT_NAME" +echo "Input data: $INPUT_DATA" +echo "Output file: $OUTPUT_FILE" +echo "Policy URL: $VLLM_URL" +[ -n "$JUDGE_URL" ] && echo "Judge URL: $JUDGE_URL" +echo "============================================================" +if [ -f "$DONE_FILE" ]; then + echo "Chunk already complete (.done exists) β€” skipping." + exit 0 +fi + +echo "" +echo "[Step 1/3] Waiting for vLLM servers ..." +wait_for_server "http://policy:8000/v1/models" "Policy vLLM" $$ 400 /dev/null +wait_for_server "http://judge:8001/v1/models" "Judge vLLM" $$ 400 /dev/null + +CHUNK_INPUT="" +if [ $NUM_CHUNKS -gt 1 ]; then + echo "" + echo "[Step 1a] Extracting chunk slice ..." + TOTAL_LINES=$(wc -l < "$INPUT_DATA") + EFFECTIVE=$TOTAL_LINES + MAX_SAMPLES=0 + if [ $MAX_SAMPLES -gt 0 ] && [ $MAX_SAMPLES -lt $TOTAL_LINES ]; then + EFFECTIVE=$MAX_SAMPLES + fi + CHUNK_SIZE=$(( (EFFECTIVE + NUM_CHUNKS - 1) / NUM_CHUNKS )) + START_LINE=$(( CHUNK_ID * CHUNK_SIZE + 1 )) + END_LINE=$(( (CHUNK_ID + 1) * CHUNK_SIZE )) + [ $END_LINE -gt $EFFECTIVE ] && END_LINE=$EFFECTIVE + CHUNK_INPUT="$OUTPUT_DIR/chunk_input_chunk$CHUNK_ID.jsonl" + head -n $END_LINE "$INPUT_DATA" | tail -n +$START_LINE > "$CHUNK_INPUT" + echo " Chunk $CHUNK_ID/$NUM_CHUNKS: lines $START_LINE-$END_LINE ($((END_LINE - START_LINE + 1)) samples)" + INPUT_DATA="$CHUNK_INPUT" +fi +ASYNC_FILE="$OUTPUT_FILE-async" +( + # Case A: output exists without .done β†’ restore to -async + if [ -f "$OUTPUT_FILE" ] && [ ! -f "$DONE_FILE" ]; then + echo "[Self-heal] Output file exists without .done β€” restoring to -async ..." + mv -f "$OUTPUT_FILE" "$ASYNC_FILE" + fi + + # Case B: orphaned .prev β†’ merge into -async + if [ -f "$ASYNC_FILE.prev" ]; then + echo "[Self-heal] Found orphaned .prev β€” merging into -async ..." + PREV_LINES=$(wc -l < "$ASYNC_FILE.prev") + ASYNC_LINES=0 + [ -f "$ASYNC_FILE" ] && ASYNC_LINES=$(wc -l < "$ASYNC_FILE") + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.healed" + [ -f "$ASYNC_FILE" ] && cat "$ASYNC_FILE" >> "$ASYNC_FILE.healed" + mv -f "$ASYNC_FILE.healed" "$ASYNC_FILE" && rm -f "$ASYNC_FILE.prev" + MERGED_LINES=$(wc -l < "$ASYNC_FILE") + echo " Recovered $PREV_LINES (prev) + $ASYNC_LINES (async) = $MERGED_LINES total rows" + fi + + # Case C: clean up orphaned temp files from prior crash + rm -f "$ASYNC_FILE.healed" "$ASYNC_FILE.restored" "$ASYNC_FILE.merged" +) || echo "[Self-heal] WARNING: recovery failed β€” continuing with available data" +REMAINING_INPUT="$OUTPUT_DIR/remaining_input_chunk$CHUNK_ID.jsonl" + +if ! PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.resume_filter "$ASYNC_FILE" "$INPUT_DATA" "$REMAINING_INPUT" 0; then + echo "ERROR: resume_filter failed" >&2 + exit 1 +fi + +if [ -f "$ASYNC_FILE" ] && [ ! -s "$REMAINING_INPUT" ]; then + echo "All rows already completed in -async -- finalizing." + cp -f "$ASYNC_FILE" "$OUTPUT_FILE" + touch "$DONE_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_FILE" "$ASYNC_FILE.prev" + echo "Done [$JOB_LABEL]." + exit 0 +fi + +HEAD_SERVER_PORT=$(find_free_port) + +cd "$GYM_PATH" + +echo "" +echo "[Step 2/3] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=$VLLM_URL" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=$MODEL_PATH" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+port_range_low=1024" \ + "+port_range_high=8999" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \ + "+judge_model.responses_api_models.vllm_model.base_url=http://judge:8001/v1" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 3/3] Collecting rollouts ..." +# Back up previous partial results before `gym eval run` clears the file. +ASYNC_BACKUP="" +PREV_MERGED=0 +if [ -s "$ASYNC_FILE" ]; then + ASYNC_BACKUP="$ASYNC_FILE.prev" + cp "$ASYNC_FILE" "$ASYNC_BACKUP" +fi +# Ensure clean slate for first attempt. Prior data is safe in .prev. +# Stale materialized_inputs from a prior Slurm job would cause +# resume_from_cache to load wrong task indexes. +rm -f "$ASYNC_FILE" +MATERIALIZED="$(dirname "$ASYNC_FILE")/$(basename "$ASYNC_FILE" .jsonl-async)_materialized_inputs.jsonl" +rm -f "$MATERIALIZED" +# Retry loop: the vLLM tokenizer race condition (RuntimeError: Already +# borrowed) can crash the client on the initial request burst. Retrying +# after a short delay shifts the timing and almost always succeeds. +# resume_from_cache=true ensures retries skip completed samples. +_NVFLOW_MAX_RETRIES=3 +_NVFLOW_RETRY_DELAY=15 +_NVFLOW_EXIT=0 +for _attempt in $(seq 1 $_NVFLOW_MAX_RETRIES); do + # Guard: truncate corrupted last line from SIGKILL mid-write + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + if [ "$(tail -c 1 "$ASYNC_FILE" | xxd -p)" != "0a" ]; then + head -n -1 "$ASYNC_FILE" > "$ASYNC_FILE.truncated" \ + && mv -f "$ASYNC_FILE.truncated" "$ASYNC_FILE" \ + || rm -f "$ASYNC_FILE.truncated" + echo "[nvflow] Truncated corrupted last line from $ASYNC_FILE" + fi + fi + set +e + gym eval run --no-serve \ + ${AGENT_NAME:++agent_name=$AGENT_NAME} \ + +input_jsonl_fpath=$REMAINING_INPUT \ + +output_jsonl_fpath=$ASYNC_FILE \ + +num_repeats=1 \ + +resume_from_cache=true \ + +num_samples_in_parallel=$NUM_PARALLEL \ + +head_server.host=127.0.0.1 \ + +head_server.port=$HEAD_SERVER_PORT \ + +responses_create_params.max_output_tokens=32768 \ + +responses_create_params.temperature=1.0 + _NVFLOW_EXIT=$? + set -e + [ $_NVFLOW_EXIT -eq 0 ] && break + # F6: `gym eval run` can exit non-zero AFTER all rollouts + # have already been written to ASYNC_FILE -- the gym's + # post-collection aggregate_metrics call returned 500 in the + # Nemotron-Nano smoke run, killing the process at 99% even + # though all 1000 rollouts were on disk. Retrying in that + # state is wasteful (re-runs everything) and dangerous if the + # container FS is being torn down (we hit + # ``/usr/bin/sleep: No such file or directory`` then). If + # ASYNC_FILE has at least as many rows as REMAINING_INPUT, we + # already have what we need; declare success and let finalize + # do its job. + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + _async_rows=$(wc -l < "$ASYNC_FILE" 2>&- || echo 0) + _input_rows=$(wc -l < "$REMAINING_INPUT" 2>&- || echo 0) + if [ "$_input_rows" -gt 0 ] && [ "$_async_rows" -ge "$_input_rows" ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT but $ASYNC_FILE has $_async_rows/$_input_rows rows -- treating as complete (post-collection error in gym, rollouts intact)." + _NVFLOW_EXIT=0 + break + fi + fi + if [ $_attempt -lt $_NVFLOW_MAX_RETRIES ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT (attempt $_attempt/$_NVFLOW_MAX_RETRIES). Retrying in ${_NVFLOW_RETRY_DELAY}s ..." + sleep $_NVFLOW_RETRY_DELAY + _NVFLOW_RETRY_DELAY=$((_NVFLOW_RETRY_DELAY * 2)) + fi +done +if [ $_NVFLOW_EXIT -ne 0 ]; then + echo "[nvflow] gym eval run failed after $_NVFLOW_MAX_RETRIES attempts." + exit $_NVFLOW_EXIT +fi + +# Merge previous partial results with new results. +if [ -n "$ASYNC_BACKUP" ] && [ -f "$ASYNC_BACKUP" ]; then + cat "$ASYNC_BACKUP" "$ASYNC_FILE" > "$ASYNC_FILE.merged" + mv -f "$ASYNC_FILE.merged" "$ASYNC_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_BACKUP" +fi +cp -f "$ASYNC_FILE" "$OUTPUT_FILE" +touch "$DONE_FILE" +# Safe to clean up β€” .done exists, chunk won't be rescheduled. +rm -f "$ASYNC_FILE" "$REMAINING_INPUT" +[ -n "$CHUNK_INPUT" ] && rm -f "$CHUNK_INPUT" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/fixtures/rollout/client_cmd_no_rcp.txt b/tests/fixtures/rollout/client_cmd_no_rcp.txt new file mode 100644 index 0000000..e6e7937 --- /dev/null +++ b/tests/fixtures/rollout/client_cmd_no_rcp.txt @@ -0,0 +1,267 @@ +set -e + +OUTPUT_DIR=/out/rollout +GYM_PATH=/opt/Gym +UV_VENV_DIR=/opt/Gym +MODEL_PATH=/hf_models/Qwen/Qwen3-30B-A3B +AGENT_NAME=finance_agent +INPUT_DATA=/data/train.jsonl +OUTPUT_FILE=/out/rollout/rs0/chunk_0.jsonl +DONE_FILE=/out/rollout/rs0/chunk_0.jsonl.done +CONFIG_PATHS=vllm.yaml,env.yaml,overlay.yaml +NUM_PARALLEL=512 +JOB_LABEL=rs0_chunk0 +VLLM_URL="http://policy:8000/v1" +JUDGE_URL="http://judge:8001/v1" +CHUNK_ID=0 +NUM_CHUNKS=8 + +mkdir -p "$OUTPUT_DIR/logs" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +NG_RUN_PID="" + +cleanup() { + local _nvflow_exit=$? + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + # Suppress stderr via ``2>&-`` (close fd) instead of + # ``2>/dev/null`` so the cleanup trap stays quiet even if + # the container's /dev/null disappeared mid-script -- which + # is exactly what happened in the Nemotron-Nano smoke run + # where pyxis tore down the container while bash was still + # in the cleanup path, producing ``/dev/null: No such file + # or directory`` noise on top of the original failure. + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>&- && wait $NG_RUN_PID 2>&- || true + # Best-effort merge of .prev into -async. On success, finalize already + # merged and removed .prev so this block is a no-op. On failure/kill, + # this is a first attempt; the self-heal at next startup is the guarantee. + # Chain with && so .prev is NEVER deleted unless the merge succeeds. + if [ -n "$ASYNC_FILE" ] && [ -f "$ASYNC_FILE.prev" ] && [ "${PREV_MERGED:-0}" -eq 0 ]; then + echo "[Cleanup] Restoring previous results into -async for resume ..." + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.restored" \ + && { [ ! -f "$ASYNC_FILE" ] || cat "$ASYNC_FILE" >> "$ASYNC_FILE.restored"; } \ + && mv -f "$ASYNC_FILE.restored" "$ASYNC_FILE" \ + && rm -f "$ASYNC_FILE.prev" \ + || echo "[Cleanup] WARNING: merge failed β€” self-heal will recover on next start" + fi + if [ $_nvflow_exit -ne 0 ]; then + echo "[nvflow] Client exited with code $_nvflow_exit β€” cancelling job ${SLURM_JOB_ID}" + scancel "${SLURM_JOB_ID}" 2>&- || kill 0 2>&- || true + fi +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Rollout Collection [$JOB_LABEL]" +echo "============================================================" +echo "Model: $MODEL_PATH" +echo "Agent: $AGENT_NAME" +echo "Input data: $INPUT_DATA" +echo "Output file: $OUTPUT_FILE" +echo "Policy URL: $VLLM_URL" +[ -n "$JUDGE_URL" ] && echo "Judge URL: $JUDGE_URL" +echo "============================================================" +if [ -f "$DONE_FILE" ]; then + echo "Chunk already complete (.done exists) β€” skipping." + exit 0 +fi + +echo "" +echo "[Step 1/3] Waiting for vLLM servers ..." +wait_for_server "http://policy:8000/v1/models" "Policy vLLM" $$ 400 /dev/null +wait_for_server "http://judge:8001/v1/models" "Judge vLLM" $$ 400 /dev/null + +CHUNK_INPUT="" +if [ $NUM_CHUNKS -gt 1 ]; then + echo "" + echo "[Step 1a] Extracting chunk slice ..." + TOTAL_LINES=$(wc -l < "$INPUT_DATA") + EFFECTIVE=$TOTAL_LINES + MAX_SAMPLES=0 + if [ $MAX_SAMPLES -gt 0 ] && [ $MAX_SAMPLES -lt $TOTAL_LINES ]; then + EFFECTIVE=$MAX_SAMPLES + fi + CHUNK_SIZE=$(( (EFFECTIVE + NUM_CHUNKS - 1) / NUM_CHUNKS )) + START_LINE=$(( CHUNK_ID * CHUNK_SIZE + 1 )) + END_LINE=$(( (CHUNK_ID + 1) * CHUNK_SIZE )) + [ $END_LINE -gt $EFFECTIVE ] && END_LINE=$EFFECTIVE + CHUNK_INPUT="$OUTPUT_DIR/chunk_input_chunk$CHUNK_ID.jsonl" + head -n $END_LINE "$INPUT_DATA" | tail -n +$START_LINE > "$CHUNK_INPUT" + echo " Chunk $CHUNK_ID/$NUM_CHUNKS: lines $START_LINE-$END_LINE ($((END_LINE - START_LINE + 1)) samples)" + INPUT_DATA="$CHUNK_INPUT" +fi +ASYNC_FILE="$OUTPUT_FILE-async" +( + # Case A: output exists without .done β†’ restore to -async + if [ -f "$OUTPUT_FILE" ] && [ ! -f "$DONE_FILE" ]; then + echo "[Self-heal] Output file exists without .done β€” restoring to -async ..." + mv -f "$OUTPUT_FILE" "$ASYNC_FILE" + fi + + # Case B: orphaned .prev β†’ merge into -async + if [ -f "$ASYNC_FILE.prev" ]; then + echo "[Self-heal] Found orphaned .prev β€” merging into -async ..." + PREV_LINES=$(wc -l < "$ASYNC_FILE.prev") + ASYNC_LINES=0 + [ -f "$ASYNC_FILE" ] && ASYNC_LINES=$(wc -l < "$ASYNC_FILE") + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.healed" + [ -f "$ASYNC_FILE" ] && cat "$ASYNC_FILE" >> "$ASYNC_FILE.healed" + mv -f "$ASYNC_FILE.healed" "$ASYNC_FILE" && rm -f "$ASYNC_FILE.prev" + MERGED_LINES=$(wc -l < "$ASYNC_FILE") + echo " Recovered $PREV_LINES (prev) + $ASYNC_LINES (async) = $MERGED_LINES total rows" + fi + + # Case C: clean up orphaned temp files from prior crash + rm -f "$ASYNC_FILE.healed" "$ASYNC_FILE.restored" "$ASYNC_FILE.merged" +) || echo "[Self-heal] WARNING: recovery failed β€” continuing with available data" +REMAINING_INPUT="$OUTPUT_DIR/remaining_input_chunk$CHUNK_ID.jsonl" + +if ! PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.resume_filter "$ASYNC_FILE" "$INPUT_DATA" "$REMAINING_INPUT" 0; then + echo "ERROR: resume_filter failed" >&2 + exit 1 +fi + +if [ -f "$ASYNC_FILE" ] && [ ! -s "$REMAINING_INPUT" ]; then + echo "All rows already completed in -async -- finalizing." + cp -f "$ASYNC_FILE" "$OUTPUT_FILE" + touch "$DONE_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_FILE" "$ASYNC_FILE.prev" + echo "Done [$JOB_LABEL]." + exit 0 +fi + +HEAD_SERVER_PORT=$(find_free_port) + +cd "$GYM_PATH" + +echo "" +echo "[Step 2/3] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=$VLLM_URL" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=$MODEL_PATH" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+port_range_low=1024" \ + "+port_range_high=8999" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \ + "+judge_model.responses_api_models.vllm_model.base_url=http://judge:8001/v1" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 3/3] Collecting rollouts ..." +# Back up previous partial results before `gym eval run` clears the file. +ASYNC_BACKUP="" +PREV_MERGED=0 +if [ -s "$ASYNC_FILE" ]; then + ASYNC_BACKUP="$ASYNC_FILE.prev" + cp "$ASYNC_FILE" "$ASYNC_BACKUP" +fi +# Ensure clean slate for first attempt. Prior data is safe in .prev. +# Stale materialized_inputs from a prior Slurm job would cause +# resume_from_cache to load wrong task indexes. +rm -f "$ASYNC_FILE" +MATERIALIZED="$(dirname "$ASYNC_FILE")/$(basename "$ASYNC_FILE" .jsonl-async)_materialized_inputs.jsonl" +rm -f "$MATERIALIZED" +# Retry loop: the vLLM tokenizer race condition (RuntimeError: Already +# borrowed) can crash the client on the initial request burst. Retrying +# after a short delay shifts the timing and almost always succeeds. +# resume_from_cache=true ensures retries skip completed samples. +_NVFLOW_MAX_RETRIES=3 +_NVFLOW_RETRY_DELAY=15 +_NVFLOW_EXIT=0 +for _attempt in $(seq 1 $_NVFLOW_MAX_RETRIES); do + # Guard: truncate corrupted last line from SIGKILL mid-write + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + if [ "$(tail -c 1 "$ASYNC_FILE" | xxd -p)" != "0a" ]; then + head -n -1 "$ASYNC_FILE" > "$ASYNC_FILE.truncated" \ + && mv -f "$ASYNC_FILE.truncated" "$ASYNC_FILE" \ + || rm -f "$ASYNC_FILE.truncated" + echo "[nvflow] Truncated corrupted last line from $ASYNC_FILE" + fi + fi + set +e + gym eval run --no-serve \ + ${AGENT_NAME:++agent_name=$AGENT_NAME} \ + +input_jsonl_fpath=$REMAINING_INPUT \ + +output_jsonl_fpath=$ASYNC_FILE \ + +num_repeats=1 \ + +resume_from_cache=true \ + +num_samples_in_parallel=$NUM_PARALLEL \ + +head_server.host=127.0.0.1 \ + +head_server.port=$HEAD_SERVER_PORT + _NVFLOW_EXIT=$? + set -e + [ $_NVFLOW_EXIT -eq 0 ] && break + # F6: `gym eval run` can exit non-zero AFTER all rollouts + # have already been written to ASYNC_FILE -- the gym's + # post-collection aggregate_metrics call returned 500 in the + # Nemotron-Nano smoke run, killing the process at 99% even + # though all 1000 rollouts were on disk. Retrying in that + # state is wasteful (re-runs everything) and dangerous if the + # container FS is being torn down (we hit + # ``/usr/bin/sleep: No such file or directory`` then). If + # ASYNC_FILE has at least as many rows as REMAINING_INPUT, we + # already have what we need; declare success and let finalize + # do its job. + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + _async_rows=$(wc -l < "$ASYNC_FILE" 2>&- || echo 0) + _input_rows=$(wc -l < "$REMAINING_INPUT" 2>&- || echo 0) + if [ "$_input_rows" -gt 0 ] && [ "$_async_rows" -ge "$_input_rows" ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT but $ASYNC_FILE has $_async_rows/$_input_rows rows -- treating as complete (post-collection error in gym, rollouts intact)." + _NVFLOW_EXIT=0 + break + fi + fi + if [ $_attempt -lt $_NVFLOW_MAX_RETRIES ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT (attempt $_attempt/$_NVFLOW_MAX_RETRIES). Retrying in ${_NVFLOW_RETRY_DELAY}s ..." + sleep $_NVFLOW_RETRY_DELAY + _NVFLOW_RETRY_DELAY=$((_NVFLOW_RETRY_DELAY * 2)) + fi +done +if [ $_NVFLOW_EXIT -ne 0 ]; then + echo "[nvflow] gym eval run failed after $_NVFLOW_MAX_RETRIES attempts." + exit $_NVFLOW_EXIT +fi + +# Merge previous partial results with new results. +if [ -n "$ASYNC_BACKUP" ] && [ -f "$ASYNC_BACKUP" ]; then + cat "$ASYNC_BACKUP" "$ASYNC_FILE" > "$ASYNC_FILE.merged" + mv -f "$ASYNC_FILE.merged" "$ASYNC_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_BACKUP" +fi +cp -f "$ASYNC_FILE" "$OUTPUT_FILE" +touch "$DONE_FILE" +# Safe to clean up β€” .done exists, chunk won't be rescheduled. +rm -f "$ASYNC_FILE" "$REMAINING_INPUT" +[ -n "$CHUNK_INPUT" ] && rm -f "$CHUNK_INPUT" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/fixtures/rollout/client_cmd_policy_only.txt b/tests/fixtures/rollout/client_cmd_policy_only.txt new file mode 100644 index 0000000..69062a3 --- /dev/null +++ b/tests/fixtures/rollout/client_cmd_policy_only.txt @@ -0,0 +1,266 @@ +set -e + +OUTPUT_DIR=/out/rollout +GYM_PATH=/opt/Gym +UV_VENV_DIR=/opt/Gym +MODEL_PATH=/hf_models/Qwen/Qwen3-30B-A3B +AGENT_NAME=finance_agent +INPUT_DATA=/data/train.jsonl +OUTPUT_FILE=/out/rollout/rs0/chunk_0.jsonl +DONE_FILE=/out/rollout/rs0/chunk_0.jsonl.done +CONFIG_PATHS=vllm.yaml,env.yaml,overlay.yaml +NUM_PARALLEL=512 +JOB_LABEL=rs0_chunk0 +VLLM_URL="http://policy:8000/v1" +JUDGE_URL="" +CHUNK_ID=0 +NUM_CHUNKS=8 + +mkdir -p "$OUTPUT_DIR/logs" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +NG_RUN_PID="" + +cleanup() { + local _nvflow_exit=$? + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + # Suppress stderr via ``2>&-`` (close fd) instead of + # ``2>/dev/null`` so the cleanup trap stays quiet even if + # the container's /dev/null disappeared mid-script -- which + # is exactly what happened in the Nemotron-Nano smoke run + # where pyxis tore down the container while bash was still + # in the cleanup path, producing ``/dev/null: No such file + # or directory`` noise on top of the original failure. + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>&- && wait $NG_RUN_PID 2>&- || true + # Best-effort merge of .prev into -async. On success, finalize already + # merged and removed .prev so this block is a no-op. On failure/kill, + # this is a first attempt; the self-heal at next startup is the guarantee. + # Chain with && so .prev is NEVER deleted unless the merge succeeds. + if [ -n "$ASYNC_FILE" ] && [ -f "$ASYNC_FILE.prev" ] && [ "${PREV_MERGED:-0}" -eq 0 ]; then + echo "[Cleanup] Restoring previous results into -async for resume ..." + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.restored" \ + && { [ ! -f "$ASYNC_FILE" ] || cat "$ASYNC_FILE" >> "$ASYNC_FILE.restored"; } \ + && mv -f "$ASYNC_FILE.restored" "$ASYNC_FILE" \ + && rm -f "$ASYNC_FILE.prev" \ + || echo "[Cleanup] WARNING: merge failed β€” self-heal will recover on next start" + fi + if [ $_nvflow_exit -ne 0 ]; then + echo "[nvflow] Client exited with code $_nvflow_exit β€” cancelling job ${SLURM_JOB_ID}" + scancel "${SLURM_JOB_ID}" 2>&- || kill 0 2>&- || true + fi +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Rollout Collection [$JOB_LABEL]" +echo "============================================================" +echo "Model: $MODEL_PATH" +echo "Agent: $AGENT_NAME" +echo "Input data: $INPUT_DATA" +echo "Output file: $OUTPUT_FILE" +echo "Policy URL: $VLLM_URL" +[ -n "$JUDGE_URL" ] && echo "Judge URL: $JUDGE_URL" +echo "============================================================" +if [ -f "$DONE_FILE" ]; then + echo "Chunk already complete (.done exists) β€” skipping." + exit 0 +fi + +echo "" +echo "[Step 1/3] Waiting for vLLM servers ..." +wait_for_server "http://policy:8000/v1/models" "Policy vLLM" $$ 400 /dev/null + +CHUNK_INPUT="" +if [ $NUM_CHUNKS -gt 1 ]; then + echo "" + echo "[Step 1a] Extracting chunk slice ..." + TOTAL_LINES=$(wc -l < "$INPUT_DATA") + EFFECTIVE=$TOTAL_LINES + MAX_SAMPLES=0 + if [ $MAX_SAMPLES -gt 0 ] && [ $MAX_SAMPLES -lt $TOTAL_LINES ]; then + EFFECTIVE=$MAX_SAMPLES + fi + CHUNK_SIZE=$(( (EFFECTIVE + NUM_CHUNKS - 1) / NUM_CHUNKS )) + START_LINE=$(( CHUNK_ID * CHUNK_SIZE + 1 )) + END_LINE=$(( (CHUNK_ID + 1) * CHUNK_SIZE )) + [ $END_LINE -gt $EFFECTIVE ] && END_LINE=$EFFECTIVE + CHUNK_INPUT="$OUTPUT_DIR/chunk_input_chunk$CHUNK_ID.jsonl" + head -n $END_LINE "$INPUT_DATA" | tail -n +$START_LINE > "$CHUNK_INPUT" + echo " Chunk $CHUNK_ID/$NUM_CHUNKS: lines $START_LINE-$END_LINE ($((END_LINE - START_LINE + 1)) samples)" + INPUT_DATA="$CHUNK_INPUT" +fi +ASYNC_FILE="$OUTPUT_FILE-async" +( + # Case A: output exists without .done β†’ restore to -async + if [ -f "$OUTPUT_FILE" ] && [ ! -f "$DONE_FILE" ]; then + echo "[Self-heal] Output file exists without .done β€” restoring to -async ..." + mv -f "$OUTPUT_FILE" "$ASYNC_FILE" + fi + + # Case B: orphaned .prev β†’ merge into -async + if [ -f "$ASYNC_FILE.prev" ]; then + echo "[Self-heal] Found orphaned .prev β€” merging into -async ..." + PREV_LINES=$(wc -l < "$ASYNC_FILE.prev") + ASYNC_LINES=0 + [ -f "$ASYNC_FILE" ] && ASYNC_LINES=$(wc -l < "$ASYNC_FILE") + cat "$ASYNC_FILE.prev" > "$ASYNC_FILE.healed" + [ -f "$ASYNC_FILE" ] && cat "$ASYNC_FILE" >> "$ASYNC_FILE.healed" + mv -f "$ASYNC_FILE.healed" "$ASYNC_FILE" && rm -f "$ASYNC_FILE.prev" + MERGED_LINES=$(wc -l < "$ASYNC_FILE") + echo " Recovered $PREV_LINES (prev) + $ASYNC_LINES (async) = $MERGED_LINES total rows" + fi + + # Case C: clean up orphaned temp files from prior crash + rm -f "$ASYNC_FILE.healed" "$ASYNC_FILE.restored" "$ASYNC_FILE.merged" +) || echo "[Self-heal] WARNING: recovery failed β€” continuing with available data" +REMAINING_INPUT="$OUTPUT_DIR/remaining_input_chunk$CHUNK_ID.jsonl" + +if ! PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.resume_filter "$ASYNC_FILE" "$INPUT_DATA" "$REMAINING_INPUT" 0; then + echo "ERROR: resume_filter failed" >&2 + exit 1 +fi + +if [ -f "$ASYNC_FILE" ] && [ ! -s "$REMAINING_INPUT" ]; then + echo "All rows already completed in -async -- finalizing." + cp -f "$ASYNC_FILE" "$OUTPUT_FILE" + touch "$DONE_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_FILE" "$ASYNC_FILE.prev" + echo "Done [$JOB_LABEL]." + exit 0 +fi + +HEAD_SERVER_PORT=$(find_free_port) + +cd "$GYM_PATH" + +echo "" +echo "[Step 2/3] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=$VLLM_URL" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=$MODEL_PATH" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+port_range_low=1024" \ + "+port_range_high=8999" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 3/3] Collecting rollouts ..." +# Back up previous partial results before `gym eval run` clears the file. +ASYNC_BACKUP="" +PREV_MERGED=0 +if [ -s "$ASYNC_FILE" ]; then + ASYNC_BACKUP="$ASYNC_FILE.prev" + cp "$ASYNC_FILE" "$ASYNC_BACKUP" +fi +# Ensure clean slate for first attempt. Prior data is safe in .prev. +# Stale materialized_inputs from a prior Slurm job would cause +# resume_from_cache to load wrong task indexes. +rm -f "$ASYNC_FILE" +MATERIALIZED="$(dirname "$ASYNC_FILE")/$(basename "$ASYNC_FILE" .jsonl-async)_materialized_inputs.jsonl" +rm -f "$MATERIALIZED" +# Retry loop: the vLLM tokenizer race condition (RuntimeError: Already +# borrowed) can crash the client on the initial request burst. Retrying +# after a short delay shifts the timing and almost always succeeds. +# resume_from_cache=true ensures retries skip completed samples. +_NVFLOW_MAX_RETRIES=3 +_NVFLOW_RETRY_DELAY=15 +_NVFLOW_EXIT=0 +for _attempt in $(seq 1 $_NVFLOW_MAX_RETRIES); do + # Guard: truncate corrupted last line from SIGKILL mid-write + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + if [ "$(tail -c 1 "$ASYNC_FILE" | xxd -p)" != "0a" ]; then + head -n -1 "$ASYNC_FILE" > "$ASYNC_FILE.truncated" \ + && mv -f "$ASYNC_FILE.truncated" "$ASYNC_FILE" \ + || rm -f "$ASYNC_FILE.truncated" + echo "[nvflow] Truncated corrupted last line from $ASYNC_FILE" + fi + fi + set +e + gym eval run --no-serve \ + ${AGENT_NAME:++agent_name=$AGENT_NAME} \ + +input_jsonl_fpath=$REMAINING_INPUT \ + +output_jsonl_fpath=$ASYNC_FILE \ + +num_repeats=1 \ + +resume_from_cache=true \ + +num_samples_in_parallel=$NUM_PARALLEL \ + +head_server.host=127.0.0.1 \ + +head_server.port=$HEAD_SERVER_PORT \ + +responses_create_params.max_output_tokens=32768 \ + +responses_create_params.temperature=1.0 + _NVFLOW_EXIT=$? + set -e + [ $_NVFLOW_EXIT -eq 0 ] && break + # F6: `gym eval run` can exit non-zero AFTER all rollouts + # have already been written to ASYNC_FILE -- the gym's + # post-collection aggregate_metrics call returned 500 in the + # Nemotron-Nano smoke run, killing the process at 99% even + # though all 1000 rollouts were on disk. Retrying in that + # state is wasteful (re-runs everything) and dangerous if the + # container FS is being torn down (we hit + # ``/usr/bin/sleep: No such file or directory`` then). If + # ASYNC_FILE has at least as many rows as REMAINING_INPUT, we + # already have what we need; declare success and let finalize + # do its job. + if [ -f "$ASYNC_FILE" ] && [ -s "$ASYNC_FILE" ]; then + _async_rows=$(wc -l < "$ASYNC_FILE" 2>&- || echo 0) + _input_rows=$(wc -l < "$REMAINING_INPUT" 2>&- || echo 0) + if [ "$_input_rows" -gt 0 ] && [ "$_async_rows" -ge "$_input_rows" ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT but $ASYNC_FILE has $_async_rows/$_input_rows rows -- treating as complete (post-collection error in gym, rollouts intact)." + _NVFLOW_EXIT=0 + break + fi + fi + if [ $_attempt -lt $_NVFLOW_MAX_RETRIES ]; then + echo "[nvflow] gym eval run exited $_NVFLOW_EXIT (attempt $_attempt/$_NVFLOW_MAX_RETRIES). Retrying in ${_NVFLOW_RETRY_DELAY}s ..." + sleep $_NVFLOW_RETRY_DELAY + _NVFLOW_RETRY_DELAY=$((_NVFLOW_RETRY_DELAY * 2)) + fi +done +if [ $_NVFLOW_EXIT -ne 0 ]; then + echo "[nvflow] gym eval run failed after $_NVFLOW_MAX_RETRIES attempts." + exit $_NVFLOW_EXIT +fi + +# Merge previous partial results with new results. +if [ -n "$ASYNC_BACKUP" ] && [ -f "$ASYNC_BACKUP" ]; then + cat "$ASYNC_BACKUP" "$ASYNC_FILE" > "$ASYNC_FILE.merged" + mv -f "$ASYNC_FILE.merged" "$ASYNC_FILE" + PREV_MERGED=1 + rm -f "$ASYNC_BACKUP" +fi +cp -f "$ASYNC_FILE" "$OUTPUT_FILE" +touch "$DONE_FILE" +# Safe to clean up β€” .done exists, chunk won't be rescheduled. +rm -f "$ASYNC_FILE" "$REMAINING_INPUT" +[ -n "$CHUNK_INPUT" ] && rm -f "$CHUNK_INPUT" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/fixtures/rollout/filter_cmd_full.txt b/tests/fixtures/rollout/filter_cmd_full.txt new file mode 100644 index 0000000..f805aaa --- /dev/null +++ b/tests/fixtures/rollout/filter_cmd_full.txt @@ -0,0 +1,14 @@ +set -e +echo "Filter Training Data (reward-variance difficulty)" +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.filter_training_data \ + "/in/train.jsonl" \ + "/out/rollout/aggregate/difficulty.jsonl" \ + "/out" \ + --min-reward-std 1e-06 \ + --train-filename "train.jsonl" \ + --val-filename "validation.jsonl" \ + --report-filename "filter_report.json" \ + --validation-data "/in/val.jsonl" \ + --policy-model "/hf_models/Qwen/Qwen3-30B-A3B" \ + --judge-model "/hf_models/openai/gpt-oss-120b" +echo "Done. Filtered data in /out/" diff --git a/tests/fixtures/rollout/filter_cmd_minimal.txt b/tests/fixtures/rollout/filter_cmd_minimal.txt new file mode 100644 index 0000000..c0fc3de --- /dev/null +++ b/tests/fixtures/rollout/filter_cmd_minimal.txt @@ -0,0 +1,11 @@ +set -e +echo "Filter Training Data (reward-variance difficulty)" +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.filter_training_data \ + "/in/train.jsonl" \ + "/out/rollout/aggregate/difficulty.jsonl" \ + "/out" \ + --min-reward-std 1e-06 \ + --train-filename "train.jsonl" \ + --val-filename "validation.jsonl" \ + --report-filename "filter_report.json" +echo "Done. Filtered data in /out/" diff --git a/tests/fixtures/rollout/merge_cmd_1chunk.txt b/tests/fixtures/rollout/merge_cmd_1chunk.txt new file mode 100644 index 0000000..ae23811 --- /dev/null +++ b/tests/fixtures/rollout/merge_cmd_1chunk.txt @@ -0,0 +1,78 @@ +set -e + +MERGED_FILE=/out/rollout/output-rs0.jsonl +ANALYSIS_DIR=/out/rollout/analysis_rs0 +SEED_LABEL=rs0 +NUM_CHUNKS=1 +INPUT_DATA=/data/train.jsonl + +echo "============================================================" +echo "Merge Rollout Chunks [$SEED_LABEL]" +echo "============================================================" + +# Clean up stale temp file from a prior crashed merge +rm -f "$MERGED_FILE.tmp" + +# Precondition: ALL chunk .done markers must exist. A missing +# .done means upstream rollout never finalised (job failed before +# the cp/touch finalize step, or output was deleted). Fail loudly +# so Slurm marks the merge job FAILED and an operator notices -- +# silently exit-0'ing here is what produced the silent-success +# cascade where aggregate runs on N-1 seeds and the pipeline +# reports COMPLETED 0:0 despite missing training data. +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + CHUNK_DONE="/out/rollout/rs0/chunk_$i.jsonl.done" + if [ ! -f "$CHUNK_DONE" ]; then + echo "ERROR: chunk $i .done missing for [$SEED_LABEL] -- upstream rollout failed before finalize. Aborting merge." + exit 1 + fi +done + +echo "[Step 1/3] Merging chunk files ..." +> "$MERGED_FILE.tmp" +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + CHUNK_FILE="/out/rollout/rs0/chunk_$i.jsonl" + if [ ! -f "$CHUNK_FILE" ] || [ ! -s "$CHUNK_FILE" ]; then + echo "ERROR: chunk $i .done exists but file missing/empty β€” aborting." + rm -f "$MERGED_FILE.tmp" + exit 1 + fi + LINES=$(wc -l < "$CHUNK_FILE") + echo " Chunk $i: $LINES lines" + cat "$CHUNK_FILE" >> "$MERGED_FILE.tmp" +done + +TOTAL=$(wc -l < "$MERGED_FILE.tmp") +echo " Merged total: $TOTAL lines" +if [ "$TOTAL" -eq 0 ]; then + echo "ERROR: All chunks present but 0 lines merged." + rm -f "$MERGED_FILE.tmp" + exit 1 +fi + +# Atomic replace β€” old merged file untouched until this point +mv -f "$MERGED_FILE.tmp" "$MERGED_FILE" + +echo "" +echo "[Step 2/3] Enriching rollouts with input metadata ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.enrich_rollouts \ + "$INPUT_DATA" \ + "$MERGED_FILE" + +echo "" +echo "[Step 3/3] Analyzing rollouts ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.analyze_rollouts \ + "$MERGED_FILE" \ + "$ANALYSIS_DIR" + +touch /out/rollout/output-rs0.jsonl.done + +# Safe cleanup: delete chunk data files only (keep .done markers). +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + CHUNK_FILE="/out/rollout/rs0/chunk_$i.jsonl" + rm -f "$CHUNK_FILE" +done +echo "Done [$SEED_LABEL]." +echo "" +echo "To browse rollouts interactively (in the nemo-gym container):" +echo " export PATH=/opt/gym-cli-venv/bin:$PATH && ng_viewer +jsonl_fpath=$MERGED_FILE" diff --git a/tests/fixtures/rollout/merge_cmd_8chunks.txt b/tests/fixtures/rollout/merge_cmd_8chunks.txt new file mode 100644 index 0000000..82e18f2 --- /dev/null +++ b/tests/fixtures/rollout/merge_cmd_8chunks.txt @@ -0,0 +1,78 @@ +set -e + +MERGED_FILE=/out/rollout/output-rs0.jsonl +ANALYSIS_DIR=/out/rollout/analysis_rs0 +SEED_LABEL=rs0 +NUM_CHUNKS=8 +INPUT_DATA=/data/train.jsonl + +echo "============================================================" +echo "Merge Rollout Chunks [$SEED_LABEL]" +echo "============================================================" + +# Clean up stale temp file from a prior crashed merge +rm -f "$MERGED_FILE.tmp" + +# Precondition: ALL chunk .done markers must exist. A missing +# .done means upstream rollout never finalised (job failed before +# the cp/touch finalize step, or output was deleted). Fail loudly +# so Slurm marks the merge job FAILED and an operator notices -- +# silently exit-0'ing here is what produced the silent-success +# cascade where aggregate runs on N-1 seeds and the pipeline +# reports COMPLETED 0:0 despite missing training data. +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + CHUNK_DONE="/out/rollout/rs0/chunk_$i.jsonl.done" + if [ ! -f "$CHUNK_DONE" ]; then + echo "ERROR: chunk $i .done missing for [$SEED_LABEL] -- upstream rollout failed before finalize. Aborting merge." + exit 1 + fi +done + +echo "[Step 1/3] Merging chunk files ..." +> "$MERGED_FILE.tmp" +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + CHUNK_FILE="/out/rollout/rs0/chunk_$i.jsonl" + if [ ! -f "$CHUNK_FILE" ] || [ ! -s "$CHUNK_FILE" ]; then + echo "ERROR: chunk $i .done exists but file missing/empty β€” aborting." + rm -f "$MERGED_FILE.tmp" + exit 1 + fi + LINES=$(wc -l < "$CHUNK_FILE") + echo " Chunk $i: $LINES lines" + cat "$CHUNK_FILE" >> "$MERGED_FILE.tmp" +done + +TOTAL=$(wc -l < "$MERGED_FILE.tmp") +echo " Merged total: $TOTAL lines" +if [ "$TOTAL" -eq 0 ]; then + echo "ERROR: All chunks present but 0 lines merged." + rm -f "$MERGED_FILE.tmp" + exit 1 +fi + +# Atomic replace β€” old merged file untouched until this point +mv -f "$MERGED_FILE.tmp" "$MERGED_FILE" + +echo "" +echo "[Step 2/3] Enriching rollouts with input metadata ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.enrich_rollouts \ + "$INPUT_DATA" \ + "$MERGED_FILE" + +echo "" +echo "[Step 3/3] Analyzing rollouts ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.analyze_rollouts \ + "$MERGED_FILE" \ + "$ANALYSIS_DIR" + +touch /out/rollout/output-rs0.jsonl.done + +# Safe cleanup: delete chunk data files only (keep .done markers). +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + CHUNK_FILE="/out/rollout/rs0/chunk_$i.jsonl" + rm -f "$CHUNK_FILE" +done +echo "Done [$SEED_LABEL]." +echo "" +echo "To browse rollouts interactively (in the nemo-gym container):" +echo " export PATH=/opt/gym-cli-venv/bin:$PATH && ng_viewer +jsonl_fpath=$MERGED_FILE" diff --git a/tests/fixtures/verify/analysis_cmd_empty_entries.txt b/tests/fixtures/verify/analysis_cmd_empty_entries.txt new file mode 100644 index 0000000..015d80f --- /dev/null +++ b/tests/fixtures/verify/analysis_cmd_empty_entries.txt @@ -0,0 +1,6 @@ +set -e +echo "Reward Analysis" +echo "Done. Analysis complete." +echo "" +echo "To browse re-judged rollouts interactively (in the nemo-gym container):" +echo " export PATH=/opt/gym-cli-venv/bin:$PATH && ng_viewer +jsonl_fpath=/out/verify/rejudge/output-rs0.jsonl" diff --git a/tests/fixtures/verify/analysis_cmd_multi_seed.txt b/tests/fixtures/verify/analysis_cmd_multi_seed.txt new file mode 100644 index 0000000..f3f5aaa --- /dev/null +++ b/tests/fixtures/verify/analysis_cmd_multi_seed.txt @@ -0,0 +1,16 @@ +set -e +echo "Reward Analysis" +echo "Analyzing rs0 ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.analyze_rollouts \ + "/out/verify/rejudge/output-rs0.jsonl" \ + "/out/verify/rejudge/analysis_rs0" \ + "REWARD RE-COMPUTATION ANALYSIS" +echo "Analyzing rs1 ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.analyze_rollouts \ + "/out/verify/rejudge/output-rs1.jsonl" \ + "/out/verify/rejudge/analysis_rs1" \ + "REWARD RE-COMPUTATION ANALYSIS" +echo "Done. Analysis complete." +echo "" +echo "To browse re-judged rollouts interactively (in the nemo-gym container):" +echo " export PATH=/opt/gym-cli-venv/bin:$PATH && ng_viewer +jsonl_fpath=/out/verify/rejudge/output-rs0.jsonl" diff --git a/tests/fixtures/verify/analysis_cmd_single_seed.txt b/tests/fixtures/verify/analysis_cmd_single_seed.txt new file mode 100644 index 0000000..1068466 --- /dev/null +++ b/tests/fixtures/verify/analysis_cmd_single_seed.txt @@ -0,0 +1,11 @@ +set -e +echo "Reward Analysis" +echo "Analyzing rs0 ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.recipes.finance.utils.rl.analyze_rollouts \ + "/out/verify/rejudge/output-rs0.jsonl" \ + "/out/verify/rejudge/analysis_rs0" \ + "REWARD RE-COMPUTATION ANALYSIS" +echo "Done. Analysis complete." +echo "" +echo "To browse re-judged rollouts interactively (in the nemo-gym container):" +echo " export PATH=/opt/gym-cli-venv/bin:$PATH && ng_viewer +jsonl_fpath=/out/verify/rejudge/output-rs0.jsonl" diff --git a/tests/fixtures/verify/verify_cmd_local_judge.txt b/tests/fixtures/verify/verify_cmd_local_judge.txt new file mode 100644 index 0000000..4c1073d --- /dev/null +++ b/tests/fixtures/verify/verify_cmd_local_judge.txt @@ -0,0 +1,94 @@ +set -e + +OUTPUT_DIR="/out/verify" +GYM_PATH="/opt/Gym" +UV_VENV_DIR="/opt/Gym" +INPUT_FILE="/in/rollouts/output-rs0.jsonl" +OUTPUT_FILE="/out/verify/rejudge/output-rs0.jsonl" +DONE_FILE="/out/verify/rejudge/output-rs0.jsonl.done" +CONFIG_PATHS="vllm.yaml,env.yaml,overlay.yaml" +NUM_PARALLEL="8" +JOB_LABEL="rejudge_rs0" +ENVIRONMENT_NAME="finance_env" + +mkdir -p "$OUTPUT_DIR/logs" "$OUTPUT_DIR/rejudge" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +HEAD_SERVER_PORT=$(find_free_port) + +NG_RUN_PID="" + +cleanup() { + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>/dev/null && wait $NG_RUN_PID 2>/dev/null || true +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Compute Rewards (re-judge) [$JOB_LABEL]" +echo "============================================================" +echo "Input file: $INPUT_FILE" +echo "Output file: $OUTPUT_FILE" +echo "Judge mode: local_vllm" +echo "Environment: $ENVIRONMENT_NAME" +echo "============================================================" + +cd "$GYM_PATH" + +echo "" +echo "[Step 1/2] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=http://localhost:0/v1" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=unused" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + "+judge_model.responses_api_models.vllm_model.entrypoint=app.py" \ + "+judge_model.responses_api_models.vllm_model.base_url=http://127.0.0.1:$JUDGE_PORT/v1" \ + "+judge_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+judge_model.responses_api_models.vllm_model.model=/hf_models/openai/gpt-oss-120b" \ + "+judge_model.responses_api_models.vllm_model.return_token_id_information=false" \ + "+judge_model.responses_api_models.vllm_model.uses_reasoning_parser=true" \ + "+finance_env.resources_servers.finance_env.judge_model_server.name=judge_model" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 2/2] Re-judging rollouts ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.verify_worker \ + "$INPUT_FILE" \ + "$OUTPUT_FILE-async" \ + "127.0.0.1" \ + "$HEAD_SERVER_PORT" \ + "$ENVIRONMENT_NAME" \ + "$NUM_PARALLEL" + +mv "$OUTPUT_FILE-async" "$OUTPUT_FILE" +touch "$DONE_FILE" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/fixtures/verify/verify_cmd_openai_judge.txt b/tests/fixtures/verify/verify_cmd_openai_judge.txt new file mode 100644 index 0000000..834a268 --- /dev/null +++ b/tests/fixtures/verify/verify_cmd_openai_judge.txt @@ -0,0 +1,91 @@ +set -e + +OUTPUT_DIR="/out/verify" +GYM_PATH="/opt/Gym" +UV_VENV_DIR="/opt/Gym" +INPUT_FILE="/in/rollouts/output-rs0.jsonl" +OUTPUT_FILE="/out/verify/rejudge/output-rs0.jsonl" +DONE_FILE="/out/verify/rejudge/output-rs0.jsonl.done" +CONFIG_PATHS="vllm.yaml,env.yaml,overlay.yaml" +NUM_PARALLEL="8" +JOB_LABEL="rejudge_rs0" +ENVIRONMENT_NAME="finance_env" + +mkdir -p "$OUTPUT_DIR/logs" "$OUTPUT_DIR/rejudge" + +find_free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('',0)); print(s.getsockname()[1]); s.close()" +} + +HEAD_SERVER_PORT=$(find_free_port) + +NG_RUN_PID="" + +cleanup() { + echo "" + echo "[Cleanup] Shutting down NeMo-Gym servers ..." + [ -n "$NG_RUN_PID" ] && kill $NG_RUN_PID 2>/dev/null && wait $NG_RUN_PID 2>/dev/null || true +} +trap cleanup EXIT + +wait_for_server() { + local url="$1" name="$2" pid="$3" max_attempts="$4" log="$5" + echo " Waiting for $name at $url ..." + for i in $(seq 1 $max_attempts); do + if curl -s -m 5 "$url" > /dev/null 2>&1; then + echo " $name ready after $((i * 5))s" + return 0 + fi + if ! kill -0 $pid 2>/dev/null; then + echo "ERROR: $name died. Check $log" + exit 1 + fi + sleep 5 + done + echo "ERROR: $name did not start within $((max_attempts * 5))s" + exit 1 +} + +echo "============================================================" +echo "Compute Rewards (re-judge) [$JOB_LABEL]" +echo "============================================================" +echo "Input file: $INPUT_FILE" +echo "Output file: $OUTPUT_FILE" +echo "Judge mode: openai" +echo "Environment: $ENVIRONMENT_NAME" +echo "============================================================" + +cd "$GYM_PATH" + +echo "" +echo "[Step 1/2] Starting NeMo-Gym servers ..." +gym env start "+config_paths=[$CONFIG_PATHS]" \ + "+policy_model.responses_api_models.vllm_model.base_url=http://localhost:0/v1" \ + "+policy_model.responses_api_models.vllm_model.api_key=EMPTY" \ + "+policy_model.responses_api_models.vllm_model.model=unused" \ + "+head_server.host=127.0.0.1" \ + "+head_server.port=$HEAD_SERVER_PORT" \ + "+skip_venv_if_present=true" \ + "+uv_venv_dir=$UV_VENV_DIR" \ + "+judge_model.responses_api_models.openai_model.base_url=https://api.openai.com/v1" \ + "+judge_model.responses_api_models.openai_model.api_key_env_var=OPENAI_API_KEY" \ + "+judge_model.responses_api_models.openai_model.model=gpt-4o-mini" \ + "+finance_env.resources_servers.finance_env.judge_model_server.name=judge_model" \ + > "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" 2>&1 & +NG_RUN_PID=$! + +wait_for_server "http://127.0.0.1:$HEAD_SERVER_PORT/" "NeMo-Gym" $NG_RUN_PID 60 "$OUTPUT_DIR/logs/ng_run_$JOB_LABEL.log" + +echo "" +echo "[Step 2/2] Re-judging rollouts ..." +PYTHONPATH=/nemo_run/code python3 -m nvflow.lib.rl.verify_worker \ + "$INPUT_FILE" \ + "$OUTPUT_FILE-async" \ + "127.0.0.1" \ + "$HEAD_SERVER_PORT" \ + "$ENVIRONMENT_NAME" \ + "$NUM_PARALLEL" + +mv "$OUTPUT_FILE-async" "$OUTPUT_FILE" +touch "$DONE_FILE" +echo "Done [$JOB_LABEL]. Cleanup via trap." diff --git a/tests/requirements-ci.txt b/tests/requirements-ci.txt new file mode 100644 index 0000000..2463d0c --- /dev/null +++ b/tests/requirements-ci.txt @@ -0,0 +1,18 @@ +# Lightweight test environment, shared by .gitlab-ci.yml and +# .github/workflows/unit-tests.yml so the two pipelines cannot drift. +# +# Installed alongside the project itself (`uv pip install -e . --no-deps`), +# which keeps out the heavy core stack (nemo-skills, torch, ~200 packages). +# Tests that genuinely need nemo-skills call pytest.importorskip and are +# skipped in both pipelines; they run in the full-deps environment instead. +# +# Add new test-only dependencies here, not to an individual CI file. +pytest>=9.0.3 +pytest-cov>=4.1.0 +pytest-timeout>=2.2.0 +PyYAML +omegaconf +rich +orjson +pandas +pyarrow diff --git a/tests/test_aggregate_seeds.py b/tests/test_aggregate_seeds.py new file mode 100644 index 0000000..6594cd4 --- /dev/null +++ b/tests/test_aggregate_seeds.py @@ -0,0 +1,135 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for nvflow.recipes.finance.utils.rl.aggregate_seeds. + +Pins the F2 contract: when ``--expected-seeds N`` is provided, missing +per-seed rollout files surface as a loud RuntimeError instead of +silently shrinking ``num_seeds`` in metrics.json. + +This is the second line of defense against the silent-success cascade +documented in the F1 commit message. The cluster's default Slurm dep +type is ``afterany`` (see cluster_configs/template-slurm.yaml note on +dependency_type), so a FAILED upstream merge does NOT prevent +aggregate from running. Without this validation, aggregate would +glob whatever ``output-rs*.jsonl`` files happened to be present and +proceed with the diminished set, producing partial difficulty data +that filter then passes through silently. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from nvflow.recipes.finance.utils.rl.aggregate_seeds import aggregate + + +def _write_seed_file(path: Path, num_rows: int = 3) -> None: + """Emit a minimal rollout file with the fields aggregate inspects.""" + rows = [ + {"uuid": f"q-{i}", "reward": float(i % 2), "question_type": "test"} for i in range(num_rows) + ] + path.write_text("".join(json.dumps(r) + "\n" for r in rows)) + + +def test_aggregate_passes_when_expected_seeds_matches_found(tmp_path: Path) -> None: + rollout_dir = tmp_path / "rollout" + rollout_dir.mkdir() + for s in range(3): + _write_seed_file(rollout_dir / f"output-rs{s}.jsonl") + out_dir = tmp_path / "agg" + + aggregate(str(rollout_dir), str(out_dir), expected_seeds=3) + + metrics = json.loads((out_dir / "metrics.json").read_text()) + assert metrics["num_seeds"] == 3 + + +def test_aggregate_raises_when_seed_missing(tmp_path: Path) -> None: + """Production scenario: rs0's merge job FAILED so output-rs0.jsonl is + absent, but rs1 and rs2 succeeded. Without --expected-seeds the + pre-F2 behaviour was to silently set num_seeds=2 and exit 0. With + F2 plumbed through (build_aggregate_cmd always passes + p.num_random_seeds), this raises so Slurm marks the aggregate job + FAILED -- visible signal that an upstream merge dropped a seed. + """ + rollout_dir = tmp_path / "rollout" + rollout_dir.mkdir() + # Only seeds 1 and 2 -- seed 0 missing. + _write_seed_file(rollout_dir / "output-rs1.jsonl") + _write_seed_file(rollout_dir / "output-rs2.jsonl") + out_dir = tmp_path / "agg" + + with pytest.raises(RuntimeError, match="Expected 3 per-seed rollout files"): + aggregate(str(rollout_dir), str(out_dir), expected_seeds=3) + + # No partial output should be written when the precondition fails. + assert not (out_dir / "metrics.json").exists() + assert not (out_dir / "summary.txt").exists() + + +def test_aggregate_raises_when_extra_seeds_present(tmp_path: Path) -> None: + """Symmetric guard: extra files (e.g. a stale output-rs7.jsonl from a + larger previous run) also fail the precondition. Otherwise an + operator could silently aggregate a mix of fresh + stale data. + """ + rollout_dir = tmp_path / "rollout" + rollout_dir.mkdir() + for s in range(5): # 5 seeds present but config expects 3 + _write_seed_file(rollout_dir / f"output-rs{s}.jsonl") + out_dir = tmp_path / "agg" + + with pytest.raises(RuntimeError, match="Expected 3 per-seed rollout files"): + aggregate(str(rollout_dir), str(out_dir), expected_seeds=3) + + +def test_aggregate_back_compat_no_expected_seeds(tmp_path: Path) -> None: + """Default expected_seeds=None preserves pre-F2 behaviour: aggregate + accepts whatever per-seed files exist. External callers (verify.py + rejudge path, ad-hoc scripts) that don't know the expected count + must continue to work. + """ + rollout_dir = tmp_path / "rollout" + rollout_dir.mkdir() + _write_seed_file(rollout_dir / "output-rs1.jsonl") + _write_seed_file(rollout_dir / "output-rs2.jsonl") + out_dir = tmp_path / "agg" + + aggregate(str(rollout_dir), str(out_dir)) # no expected_seeds kwarg + + metrics = json.loads((out_dir / "metrics.json").read_text()) + assert metrics["num_seeds"] == 2 # accepts the diminished set + + +def test_aggregate_excludes_chunk_and_async_files(tmp_path: Path) -> None: + """The seed-count comparison must be on canonical merged outputs + only. Per-chunk intermediates (output-rs0_chunk_0.jsonl) and + in-flight async files (output-rs0.jsonl-async) must be excluded + BEFORE the precondition check, otherwise stale intermediates + could mask a truly-missing seed. + """ + rollout_dir = tmp_path / "rollout" + rollout_dir.mkdir() + _write_seed_file(rollout_dir / "output-rs1.jsonl") + _write_seed_file(rollout_dir / "output-rs2.jsonl") + # Spurious intermediates that happen to glob-match. + _write_seed_file(rollout_dir / "output-rs0_chunk_0.jsonl") + _write_seed_file(rollout_dir / "output-rs0.jsonl-async") + out_dir = tmp_path / "agg" + + with pytest.raises(RuntimeError, match="found 2"): + aggregate(str(rollout_dir), str(out_dir), expected_seeds=3) diff --git a/tests/test_apply_validate_filter.py b/tests/test_apply_validate_filter.py new file mode 100644 index 0000000..7b6a05b --- /dev/null +++ b/tests/test_apply_validate_filter.py @@ -0,0 +1,196 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for ``apply_validate_filter`` post-S1. + +Pins the new contracts: +- ``raw_sdg_path`` is required (TypeError if omitted). +- ``--raw_sdg_source`` CLI flag is required (argparse exits non-zero). +- A VALID row whose ``problem`` is missing from the SDG file raises + :class:`MissingSdgRecordError` instead of silently emitting LLM bytes. +- An INVALID/missing-tag row whose ``problem`` is missing from the SDG + file is fine -- those rows go to the dropped stream and don't need + SDG-original bytes. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import orjson +import pytest + +from nvflow.recipes.finance.utils.rl.apply_validate_filter import ( + MissingSdgRecordError, + apply_validate_filter, +) + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.write_bytes(b"\n".join(orjson.dumps(r) for r in rows) + b"\n") + + +def _make_sdg_and_parsed( + tmp_path: Path, + *, + sdg_problems: list[str], + parsed: list[dict], +) -> tuple[Path, Path]: + sdg = tmp_path / "sdg.jsonl" + parsed_file = tmp_path / "parsed.jsonl" + _write_jsonl( + sdg, + [ + { + "problem": p, + "company_name": f"Co{p}", + "answer": f"A{p}", + "reasoning_content": f"R{p}", + } + for p in sdg_problems + ], + ) + _write_jsonl(parsed_file, parsed) + return sdg, parsed_file + + +# --------------------------------------------------------------------------- +# Required raw_sdg_path +# --------------------------------------------------------------------------- + + +def test_function_call_omitting_raw_sdg_path_raises_typeerror(tmp_path: Path) -> None: + """Calling the function without raw_sdg_path is a programming bug.""" + parsed_file = tmp_path / "parsed.jsonl" + parsed_file.write_bytes(b"") + with pytest.raises(TypeError, match="raw_sdg_path"): + apply_validate_filter( # type: ignore[call-arg] + input_file=str(parsed_file), + output_kept=str(tmp_path / "kept.jsonl"), + output_dropped=str(tmp_path / "dropped.jsonl"), + stats_file=str(tmp_path / "stats.json"), + ) + + +def test_cli_omitting_raw_sdg_source_exits_nonzero(tmp_path: Path) -> None: + """The CLI must require --raw_sdg_source so operators can't accidentally + fall back to a removed Mode B path. + """ + parsed_file = tmp_path / "parsed.jsonl" + parsed_file.write_bytes(b"") + proc = subprocess.run( + [ + sys.executable, + "-m", + "nvflow.recipes.finance.utils.rl.apply_validate_filter", + "--input_file", + str(parsed_file), + "--output_kept", + str(tmp_path / "kept.jsonl"), + "--output_dropped", + str(tmp_path / "dropped.jsonl"), + "--stats_file", + str(tmp_path / "stats.json"), + ], + capture_output=True, + text=True, + cwd=Path(__file__).resolve().parent.parent, + ) + assert proc.returncode != 0 + assert "--raw_sdg_source" in proc.stderr + + +# --------------------------------------------------------------------------- +# Missing-SDG-record contract +# --------------------------------------------------------------------------- + + +def test_valid_row_missing_from_sdg_raises(tmp_path: Path) -> None: + """A VALID row with no SDG counterpart is a Phase 1/2 mismatch -- raise.""" + sdg, parsed = _make_sdg_and_parsed( + tmp_path, + sdg_problems=["p_in_sdg"], + parsed=[ + {"problem": "p_NOT_in_sdg", "validate_tag": "VALID", "generation": "x"}, + ], + ) + with pytest.raises(MissingSdgRecordError, match="p_NOT_in_sdg"): + apply_validate_filter( + input_file=str(parsed), + output_kept=str(tmp_path / "kept.jsonl"), + output_dropped=str(tmp_path / "dropped.jsonl"), + stats_file=str(tmp_path / "stats.json"), + raw_sdg_path=str(sdg), + ) + + +def test_invalid_row_missing_from_sdg_is_fine(tmp_path: Path) -> None: + """Non-VALID rows go to the dropped stream regardless of SDG presence.""" + sdg, parsed = _make_sdg_and_parsed( + tmp_path, + sdg_problems=["p_in_sdg"], + parsed=[ + {"problem": "p_NOT_in_sdg", "validate_tag": "INVALID", "generation": "x"}, + {"problem": "p_NOT_in_sdg", "validate_tag": None, "generation": "y"}, + ], + ) + apply_validate_filter( + input_file=str(parsed), + output_kept=str(tmp_path / "kept.jsonl"), + output_dropped=str(tmp_path / "dropped.jsonl"), + stats_file=str(tmp_path / "stats.json"), + raw_sdg_path=str(sdg), + ) + stats = orjson.loads((tmp_path / "stats.json").read_bytes()) + assert stats["num_total"] == 2 + assert stats["num_kept"] == 0 + assert stats["num_dropped"] == 2 + # No VALID rows -> no missing-record exposure. + + +# --------------------------------------------------------------------------- +# Happy path: kept rows are byte-identical to SDG +# --------------------------------------------------------------------------- + + +def test_kept_rows_are_sdg_bytes_verbatim(tmp_path: Path) -> None: + sdg, parsed = _make_sdg_and_parsed( + tmp_path, + sdg_problems=["p1", "p2", "p3"], + parsed=[ + {"problem": "p1", "validate_tag": "VALID", "generation": "x"}, + {"problem": "p2", "validate_tag": "INVALID", "generation": "y"}, + {"problem": "p3", "validate_tag": "VALID", "generation": "z"}, + ], + ) + kept_file = tmp_path / "kept.jsonl" + apply_validate_filter( + input_file=str(parsed), + output_kept=str(kept_file), + output_dropped=str(tmp_path / "dropped.jsonl"), + stats_file=str(tmp_path / "stats.json"), + raw_sdg_path=str(sdg), + ) + + sdg_bytes_by_problem: dict[str, bytes] = {} + for line in sdg.read_bytes().splitlines(): + if line.strip(): + sdg_bytes_by_problem[orjson.loads(line)["problem"]] = line.strip() + + kept_lines = [line for line in kept_file.read_bytes().splitlines() if line.strip()] + assert len(kept_lines) == 2 + assert kept_lines[0] == sdg_bytes_by_problem["p1"] + assert kept_lines[1] == sdg_bytes_by_problem["p3"] diff --git a/tests/test_cli_cmd.py b/tests/test_cli_cmd.py new file mode 100644 index 0000000..f70c59f --- /dev/null +++ b/tests/test_cli_cmd.py @@ -0,0 +1,235 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for nvflow.lib.cli_cmd.build_python_cmd. + +Pins the shlex-quoting + interpreter contract so future refactors can't +accidentally reintroduce a path-injection vector via the rendered shell +command. This helper is shared across multiple stages +(validate_questions, data_transformation, apply_prompt_template, +convert_to_responses_api, prepare_data, prefetch_cache) so a regression +here would fan out across the entire data pipeline. +""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +from nvflow.lib.cli_cmd import build_python_cmd, build_python_script_cmd + + +def test_emits_module_invocation() -> None: + out = build_python_cmd("foo.bar.baz", input_file="/a/b.jsonl") + assert out.startswith("python3 -m foo.bar.baz ") + # Should be a single space-joined string. + assert "\n" not in out + assert " " not in out # no doubled spaces + + +def test_renders_simple_paths_unchanged() -> None: + """Plain paths (no special chars) shouldn't gain extra quotes -- shlex.quote + only quotes when needed. This makes log-grepping the rendered command + straightforward. + """ + out = build_python_cmd( + "m", input_file="/lustre/foo/bar.jsonl", stats_file="/lustre/foo/stats.json" + ) + assert "--input_file /lustre/foo/bar.jsonl" in out + assert "--stats_file /lustre/foo/stats.json" in out + # No surrounding single quotes around the safe paths. + assert "'/lustre/foo/bar.jsonl'" not in out + + +def test_quotes_path_with_spaces() -> None: + """Spaces in a path must be properly quoted so the shell parses one arg.""" + out = build_python_cmd("m", input_file="/a path/with spaces.jsonl") + # shlex.quote surrounds the value with single quotes when needed. + assert "--input_file '/a path/with spaces.jsonl'" in out + # And shlex.split must round-trip to the original token. + tokens = shlex.split(out) + idx = tokens.index("--input_file") + assert tokens[idx + 1] == "/a path/with spaces.jsonl" + + +def test_quotes_path_with_single_quote() -> None: + """Single-quote in a value is the canonical injection vector for naive + f-string command builders -- shlex.quote handles it correctly. + """ + out = build_python_cmd("m", input_file="/a/file's name.jsonl") + tokens = shlex.split(out) + idx = tokens.index("--input_file") + assert tokens[idx + 1] == "/a/file's name.jsonl" + + +def test_quotes_shell_metacharacters() -> None: + """``$``, ``;``, ``|``, backticks, ``&`` etc. must not be interpreted by + the shell as control characters when they appear in a value. + """ + dangerous = "/path; rm -rf /;$(echo pwned)`evil`|cat&" + out = build_python_cmd("m", input_file=dangerous) + tokens = shlex.split(out) + idx = tokens.index("--input_file") + assert tokens[idx + 1] == dangerous + + +def test_accepts_pathlib_values() -> None: + """Stage code passes pathlib.Path -- helper must stringify them.""" + out = build_python_cmd("m", input_file=Path("/a/b.jsonl")) + assert "--input_file /a/b.jsonl" in out + + +def test_accepts_numeric_values() -> None: + """Stages pass ints (e.g. ``num_chunks=10``) and floats (e.g. + ``context_min_percentile=1.0``) directly -- helper must stringify them. + """ + out = build_python_cmd("m", num_chunks=10, context_min_percentile=1.0) + assert "--num_chunks 10" in out + assert "--context_min_percentile 1.0" in out + + +def test_preserves_flag_order() -> None: + """Ordered output keeps log-grep diffs minimal across reruns. Python + 3.7+ preserves kwarg order so this comes for free, but the test pins it. + """ + out = build_python_cmd("m", alpha="1", beta="2", gamma="3") + assert out.index("--alpha") < out.index("--beta") < out.index("--gamma") + + +def test_handles_empty_value() -> None: + """Empty string still needs quoting so the flag's value isn't lost.""" + out = build_python_cmd("m", input_file="") + tokens = shlex.split(out) + idx = tokens.index("--input_file") + assert tokens[idx + 1] == "" + + +def test_positional_args_emitted_before_flags() -> None: + """Positional inputs (e.g. dataset_transformer's input_files) must appear + between ``-m `` and the first flag, in argument order. + """ + out = build_python_cmd( + "m", + "/a/in1.jsonl", + "/a/in2.jsonl", + output_file="/a/out.jsonl", + ) + tokens = shlex.split(out) + assert tokens[:3] == ["python3", "-m", "m"] + assert tokens[3] == "/a/in1.jsonl" + assert tokens[4] == "/a/in2.jsonl" + assert tokens[5] == "--output_file" + assert tokens[6] == "/a/out.jsonl" + + +def test_positional_args_quoted() -> None: + """Positional args must use the same shlex.quote treatment as flag values + so ``input_files`` containing spaces or metacharacters do not break. + """ + out = build_python_cmd("m", "/a path/file.jsonl", "/b/'evil'.jsonl") + tokens = shlex.split(out) + assert tokens[3] == "/a path/file.jsonl" + assert tokens[4] == "/b/'evil'.jsonl" + + +def test_positional_args_accept_pathlib() -> None: + """Stage code may pass Path positionals -- they must be stringified.""" + out = build_python_cmd("m", Path("/a/b.jsonl"), output_file=Path("/a/out.jsonl")) + tokens = shlex.split(out) + # Layout: python3 -m m /a/b.jsonl --output_file /a/out.jsonl + # [0] [1] [2] [3] [4] [5] + assert tokens[3] == "/a/b.jsonl" + assert tokens[4] == "--output_file" + assert tokens[5] == "/a/out.jsonl" + + +def test_no_positional_no_flags_renders_bare_module() -> None: + """Bare module invocation (no args at all) is a valid edge case -- + the helper should not append trailing spaces or empty tokens. + """ + out = build_python_cmd("m") + assert out == "python3 -m m" + + +def test_uses_python3_interpreter_not_python() -> None: + """All cluster containers in this repo use ``python3`` -- standardising + avoids ambiguity around the unversioned ``python`` symlink (absent in + some minimal images). Pin this so future refactors don't silently + flip back to ``python``. + """ + out = build_python_cmd("any.module") + assert out.startswith("python3 ") + assert not out.startswith("python ") + + +# --- build_python_script_cmd (path-based variant for vendored external tools) + + +def test_script_cmd_emits_path_invocation() -> None: + """``python3