image-sims workflow: campaign hardening

src/sp_validation/image_sims.py

@@ -152,11 +152,15 @@ def load_catalogs(self, verbose=True):
         grids_dir = self.cfg["grids_dir"]
         num = self.cfg["num"]
         cat_name = self.cfg.get("catalog_name", "shape_catalog_cut_ngmix.fits")
+        # Grid sims live in ``{branch}_grid_{num}``; other families (e.g.
+        # blended) in ``{branch}_{num}``.
+        sims_type = self.cfg.get("sims_type", "grid")
+        suffix = f"_grid_{num}" if sims_type == "grid" else f"_{num}"
         # ``sim_names`` (incl. the unsheared reference) comes from the config's
         # branch map. The +g/-g pool estimator pairs the sheared sims directly;
         # the reference is loaded for completeness and null-test diagnostics.
         for name in self.sim_names:
-            path = f"{grids_dir}/{name}_grid_{num}/{cat_name}"
+            path = f"{grids_dir}/{name}{suffix}/{cat_name}"
             if verbose:
                 print(f"  Loading {path}")
             self.cats[name] = _load_cat(path, self.w_cols)

workflow/image_sims/config.yaml

@@ -51,6 +51,12 @@ image_sims:
   # --- simulation grid --------------------------------------------------
   sims_type: grid        # 'grid' -> *_grid_{num}; anything else -> *_{num}  [operational]
   num: 1
+  # Optional: pull exposures from a different realization than tiles (tiles
+  # from `{branch}_{num}`, exposures from `{branch}_{exp_num}`; unset ->
+  # exposures follow `num`).  Non-grid sims only.  Fabian's blended production
+  # split: tiles from _1, exposures from _5 (the clean all-real exposure set;
+  # _1.._4 carry signal-free grid-sims SP_exp cross-links).
+  # exp_num: 5
   # Branches this run requests: the unsheared reference plus the four +/-
   # sheared branches.  Injected shear is NOT set here -- it is parsed from each
   # branch's basic_info.txt by im_manifest.  [operational]
@@ -63,6 +69,11 @@ image_sims:
   # ShapePipe cfis configs (final_cat.param etc.); default is
   # {shapepipe_repo}/example/cfis_image_sims once #766 lands.  [operational]
   config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims
+  # ShapePipe config tree for the pipeline stage (run_job's -c override); owns
+  # the ngmix ini and hence METACAL_PSF.  Empty -> run_job's default,
+  # {shapepipe_repo}/example/cfis_image_sims.  Point it at a per-arm copy to
+  # A/B the metacal PSF without editing the worktree.  [operational]
+  pipeline_config_dir: ""
   psf_model: psfex       # [operational]
   n_smp: -1              # [operational]
   # Extract/calibrate scripts run from the sp_validation repo checkout (branch

workflow/rules/image_sims.smk

@@ -86,11 +86,16 @@ _OPERATIONAL_KEYS = {
     "branches",
     "shape",
     "config_dir",
+    "pipeline_config_dir",
     "psf_model",
     "n_smp",
     "extract_script",
     "calibrate_script",
 }
+# Optional keys: recognized but not required; defaults applied in code below.
+_OPTIONAL_KEYS = {
+    "exp_num",
+}
 # Structural keys: paths/identifiers the run must supply (no sensible default).
 _STRUCTURAL_KEYS = {
     "sif",
@@ -104,7 +109,11 @@ _STRUCTURAL_KEYS = {
     "tile_ids",
 }
 _ALLOWED_KEYS = (
-    _SCIENCE_KEYS | _DEPRECATED_KEYS | _OPERATIONAL_KEYS | _STRUCTURAL_KEYS
+    _SCIENCE_KEYS
+    | _DEPRECATED_KEYS
+    | _OPERATIONAL_KEYS
+    | _OPTIONAL_KEYS
+    | _STRUCTURAL_KEYS
 )
 
 _unknown = set(IMSIM) - _ALLOWED_KEYS
@@ -155,6 +164,15 @@ SIMS_TYPE = IMSIM["sims_type"]
 _SUFFIX = f"_{SIMS_TYPE}_{NUM}" if SIMS_TYPE == "grid" else f"_{NUM}"
 SIM_BASES = list(IMSIM["branches"])
 SIMS = [f"{base}{_SUFFIX}" for base in SIM_BASES]
+# Optional ``exp_num`` pulls exposures from a different realization than tiles:
+# tiles stay on ``{branch}_{num}``, exposures come from ``{branch}_{exp_num}``.
+# Non-grid sims only (grid names embed sims_type); see config.yaml for the
+# blended use case.
+EXP_NUM = IMSIM.get("exp_num", NUM)
+if SIMS_TYPE == "grid":
+    SIM_EXP_BASE = {sim: sim for sim in SIMS}
+else:
+    SIM_EXP_BASE = {f"{base}{_SUFFIX}": f"{base}_{EXP_NUM}" for base in SIM_BASES}
 MANIFEST = f"{GRIDS_BASE}/manifest.yaml"
 BUILD_MANIFEST = f"{SPV_REPO}/workflow/scripts/im_build_manifest.py"
 
@@ -168,6 +186,10 @@ MASK_CONFIG = IMSIM["mask_config"]  # e.g. config/calibration/mask_v1.X.9_im_sim
 PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py"
 # ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param).
 CONFIG_DIR = IMSIM["config_dir"]
+# ShapePipe config dir for the pipeline stage's run_job (``-c`` override; owns
+# the ngmix ini and thus METACAL_PSF).  Empty -> run_job's default.  See
+# config.yaml.
+PIPELINE_CONFIG_DIR = IMSIM["pipeline_config_dir"]
 
 # ShapePipe scripts live in the ShapePipe repo (also baked into its image).
 CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py"
@@ -335,7 +357,8 @@ rule im_init:
         run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}",
         cfis=lambda wc: f"{GRIDS_BASE}/{wc.sim}/cfis",
         sim_tiles=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_tiles",
-        sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{wc.sim}/images/SP_exp",
+        # SIM_EXP_BASE maps the tile sim to its exposure-source sim (see above).
+        sim_exp=lambda wc: f"{INPUT_SIMS_BASE}/{SIM_EXP_BASE[wc.sim]}/images/SP_exp",
     shell:
         # cfis / input_tiles / input_exp are stable read-only symlinks (used by
         # get_images, merge, extract); created here but not tracked as outputs,
@@ -372,14 +395,17 @@ rule im_pipeline:
         run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}",
         psf=IMSIM["psf_model"],
         n_smp=IMSIM["n_smp"],
+        config_flag=f"-c {PIPELINE_CONFIG_DIR}" if PIPELINE_CONFIG_DIR else "",
     resources:
-        mem_mb=16000,
+        # measured: job uses ~1 core (eff 0.9); post-#843 MaxRSS ~72MB accounting / 2.3GB live-peak; 2cpu/4GB packs ~512 jobs on the usable partition
+        mem_mb=4000,
+        cpus_per_task=2,
         runtime=720,
     shell:
         "cd {params.run_dir} && "
         "{EXEC_PIPELINE} bash {RUN_JOB} "
         "-e {wildcards.tile} -t image_sims -j {JOB_MASK} "
-        "-p {params.psf} -N {params.n_smp}"
+        "-p {params.psf} -N {params.n_smp} {params.config_flag}"
 
 
 rule im_merge:
@@ -458,10 +484,14 @@ rule im_mbias:
         ),
     output:
         results=f"{GRIDS_BASE}/results/m_bias_results.yaml",
+    resources:
+        # Blended catalogues are ~5x grid size; the snakemake 1000M default OOMs.
+        mem_mb=8000,
     params:
         cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml",
         grids_base=GRIDS_BASE,
         num=NUM,
+        sims_type=SIMS_TYPE,
         cat_name=f"shape_catalog_cut_{SHAPE}.fits",
         sif=SIF,
         sif_pipeline=SIF_PIPELINE,
@@ -547,6 +577,7 @@ rule im_mbias:
         mbias_cfg = {
             "grids_dir": params.grids_base,
             "num": params.num,
+            "sims_type": params.sims_type,
             "catalog_name": params.cat_name,
             # Injected shear: from the manifest, the single source of truth.
             "shear_amplitude": manifest["shear_amplitude"],