Image-simulation m-bias: extracted core + snakemake orchestration

Dockerfile

@@ -1,5 +1,5 @@
 # Development image with more bells and whistles
-FROM ghcr.io/cosmostat/shapepipe:develop
+FROM ghcr.io/cosmostat/shapepipe:im_sims
 
 RUN apt-get update -y --quiet --fix-missing && \
     apt-get dist-upgrade -y --quiet --fix-missing && \

config/calibration/mask_v1.X.4_im_sim.yaml

@@ -0,0 +1,70 @@
+# Config file for masking and calibration.
+# Standard cuts without coverage mask, type v1.X.4.
+
+# General parameters (can also given on command line)
+params:
+  input_path: shape_catalog_comprehensive_ngmix.hdf5
+  cmatrices: False
+  sky_regions: False
+  verbose: True
+
+# Masks
+## Using columns in 'dat' group (ShapePipe flags)
+dat:
+  # SExtractor flags
+  - col_name: FLAGS
+    label: SE FLAGS
+    kind: equal
+    value: 0
+
+  # Number of epochs
+  - col_name: N_EPOCH
+    label: r"$n_{\rm epoch}$"
+    kind: greater_equal
+    value: 2
+
+  # Magnitude range
+  - col_name: mag
+    label: mag range
+    kind: range
+    value: [15, 30]
+
+  # ngmix flags
+  - col_name: NGMIX_MOM_FAIL
+    label: "ngmix moments failure"
+    kind: equal
+    value: 0
+
+  # invalid PSF ellipticities
+  - col_name: NGMIX_ELL_PSFo_NOSHEAR_0
+    label: "bad PSF ellipticity comp 1"
+    kind: not_equal
+    value: -10
+  - col_name: NGMIX_ELL_PSFo_NOSHEAR_1
+    label: "bad PSF ellipticity comp 2"
+    kind: not_equal
+    value: -10
+
+# Metacal parameters
+metacal:
+  # Ellipticity dispersion
+  sigma_eps_prior: 0.34
+
+  # Signal-to-noise range
+  gal_snr_min: 10
+  gal_snr_max: 500
+
+  # Relative-size (hlr / hlr_psf) range
+  gal_rel_size_min: 0.5
+  gal_rel_size_max: 3
+
+  # Correct relative size for ellipticity?
+  gal_size_corr_ell: False
+
+  # Weight for global response matrix, None for unweighted mean.
+  # Unweighted for image sims: no weights anywhere in sim m-bias (#227).
+  global_R_weight: null
+
+  # Subtract additive bias (mean shear)? Use False for constant-shear
+  # image sims
+  additive_correction: False

config/calibration/mask_v1.X.9_im_sim.overlay.yaml

@@ -0,0 +1,97 @@
+# Declared delta: image-sim mask/calibration config vs the data config.
+#
+# The image-sim calibration reuses the *data* mask config
+# (mask_v1.X.9.yaml) and changes only what the sims genuinely differ on.
+# Rather than maintain a second full copy that can silently drift from the
+# base, this overlay states -- as a list of block operations on the base
+# file -- exactly which pieces the sims drop or change, and one line of why
+# for each.  `im_compose_mask.py` applies these ops to mask_v1.X.9.yaml and
+# reproduces mask_v1.X.9_im_sim.yaml byte-for-byte; a test locks that, so the
+# runtime file and this declaration cannot diverge.
+#
+# Each op anchors to a block of base text (matched verbatim, and required to
+# occur exactly once) and either drops it or replaces it.  `why` is prose for
+# the human reader; the compose ignores it.  Ordering follows the base file.
+
+base: mask_v1.X.9.yaml
+
+ops:
+  # --- params ------------------------------------------------------------
+  - why: >-
+      Sims read the ShapePipe FITS catalogue staged in the run dir, not the
+      survey-wide comprehensive HDF5 on /n17data.
+    replace: |
+      input_path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5
+    with: |
+      input_path: shape_catalog_comprehensive_ngmix.fits
+
+  # --- dat cuts ----------------------------------------------------------
+  - why: >-
+      No ShapePipe coverage/mask flags on sims: IMAFLAGS_ISO is a survey
+      artefact (external masks, bright-star haloes) the sims do not carry.
+    drop: |2
+
+        # ShapePipe flags
+        - col_name: IMAFLAGS_ISO
+          label: SP mask
+          kind: equal
+          value: 0
+
+  - why: >-
+      Same cuts, but flag the grammar: the sims run the ShapePipe-v2 PSF
+      columns (scalar G1/G2), so the comment is made explicit here.
+    replace: |2
+        # invalid PSF ellipticities
+    with: |2
+        # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components)
+
+  # --- dat_ext (post-processing / coverage masks) ------------------------
+  - why: >-
+      No coverage masks on sims: the whole dat_ext group (Stars, manual mask,
+      r-band footprint, Maximask) is survey post-processing with no analogue
+      in the simulated tiles.
+    drop: |2
+
+      ## Using columns in 'dat_ext' group (post-processing flags)
+      dat_ext:
+
+        # Stars
+        - col_name: 4_Stars
+          label: "Stars"
+          kind: equal
+          value: False
+
+        # Manual mask
+        - col_name: 8_Manual
+          label: "manual mask"
+          kind: equal
+          value: False
+
+        # r-band footprint
+        - col_name: 64_r
+          label: "r-band imaging"
+          kind: equal
+          value: False
+
+        # Maximask
+        - col_name: 1024_Maximask
+          label: "maximask"
+          kind: equal
+          value: False
+
+  # --- metacal -----------------------------------------------------------
+  - why: >-
+      Unweighted for image sims: no weights anywhere in the sim m-bias (#227),
+      and the w_des-weighted global R had only N_eff ~ 20-100 objects.
+    replace: |2
+        # Weight for global response matrix, None for unweighted mean
+        global_R_weight: w
+    with: |2
+        # Weight for global response matrix, None for unweighted mean.
+        # Unweighted for image sims: no weights anywhere in sim m-bias (#227),
+        # and the w_des-weighted R had N_eff ~ 20-100 objects.
+        global_R_weight: null
+
+        # Subtract additive bias (mean shear)? Use False for constant-shear
+        # image sims
+        additive_correction: False

config/calibration/mask_v1.X.9_im_sim.yaml

@@ -0,0 +1,77 @@
+# Config file for masking and calibration.
+# Less conservative cuts, type v1.X.9 (e.g. for matching with spectroscopic sample).
+
+# General parameters (can also given on command line)
+params:
+  input_path: shape_catalog_comprehensive_ngmix.fits
+  cmatrices: False
+  sky_regions: False
+  verbose: True
+
+# Masks
+## Using columns in 'dat' group (ShapePipe flags)
+dat:
+  # SExtractor flags
+  - col_name: FLAGS
+    label: SE FLAGS
+    kind: smaller_equal
+    value: 2
+
+  # Duplicate objects
+  - col_name: overlap
+    label: tile overlap
+    kind: equal
+    value: True
+
+  # Number of epochs
+  - col_name: N_EPOCH
+    label: r"$n_{\rm epoch}$"
+    kind: greater_equal
+    value: 1
+
+  # Magnitude range
+  - col_name: mag
+    label: mag range
+    kind: range
+    value: [15, 30]
+
+  # ngmix flags
+  - col_name: NGMIX_MCAL_TYPES_FAIL
+    label: "ngmix moments failure"
+    kind: equal
+    value: 0
+
+  # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components)
+  - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR
+    label: "bad PSF ellipticity comp 1"
+    kind: not_equal
+    value: -10
+  - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR
+    label: "bad PSF ellipticity comp 2"
+    kind: not_equal
+    value: -10
+
+# Metacal parameters
+metacal:
+  # Ellipticity dispersion
+  sigma_eps_prior: 0.34
+
+  # Signal-to-noise range
+  gal_snr_min: 5
+  gal_snr_max: 500
+
+  # Relative-size (hlr / hlr_psf) range
+  gal_rel_size_min: 0.25
+  gal_rel_size_max: 10
+
+  # Correct relative size for ellipticity?
+  gal_size_corr_ell: False
+
+  # Weight for global response matrix, None for unweighted mean.
+  # Unweighted for image sims: no weights anywhere in sim m-bias (#227),
+  # and the w_des-weighted R had N_eff ~ 20-100 objects.
+  global_R_weight: null
+
+  # Subtract additive bias (mean shear)? Use False for constant-shear
+  # image sims
+  additive_correction: False

scripts/calibration/calibrate_comprehensive_cat.py

@@ -96,8 +96,13 @@
 )
 
 # %%
+additive_correction = cm.get("additive_correction", True)
+if not additive_correction:
+    print("Additive bias correction disabled (additive_correction: False)")
+
 g_corr_mc, g_uncorr, w, mask_metacal, c, c_err = calibration.get_calibrated_m_c(
-    gal_metacal
+    gal_metacal,
+    additive_correction=additive_correction,
 )
 
 num_ok = len(g_corr_mc[0])