From cad7eb588b787a48cb94bd4a08532e4589994b9f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 17:18:42 +0200 Subject: [PATCH 01/17] feat(mask): declare healsparse + hpgeom dependencies First step of the external-healsparse-mask path (#846): the reader rasterizes maskforce healsparse products onto image pixel grids in place of internal mask generation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CfRuCQa2UHo44yp2MZDsJX --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8b259deec..5388adc86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ dependencies = [ "cs_util>=0.2.1", "galsim>=2.8", "h5py", + "healsparse", + "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", From 3bb39e3884e3fec364ae5ad4dd2d46e71507181e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 17:46:04 +0200 Subject: [PATCH 02/17] feat(mask_ext): rasterize external healsparse masks to pipeline flag images New mask_ext module (PRD in #847): reads the image WCS, evaluates the pixel grid in memory-bounded row chunks, queries the healsparse map, applies a config-driven bit->flag mapping, optionally sums the external instrument flag (matching Mask._build_final_mask semantics), and writes the standard int16 _flag.fits. Same module serves tiles and exposure CCDs; mask files and bit meanings live only in config. Example tile/exposure configs; 7 unit tests on synthetic maps (bit mapping, chunk-seam independence, RA wrap, off-map, ext-flag sum, WCS round-trip). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CfRuCQa2UHo44yp2MZDsJX --- example/cfis/config_exp_MaExt.ini | 50 +++ example/cfis/config_tile_MaExt.ini | 51 +++ .../modules/mask_ext_package/__init__.py | 71 ++++ .../modules/mask_ext_package/mask_ext.py | 319 ++++++++++++++++++ src/shapepipe/modules/mask_ext_runner.py | 117 +++++++ tests/module/test_mask_ext.py | 230 +++++++++++++ 6 files changed, 838 insertions(+) create mode 100644 example/cfis/config_exp_MaExt.ini create mode 100644 example/cfis/config_tile_MaExt.ini create mode 100644 src/shapepipe/modules/mask_ext_package/__init__.py create mode 100644 src/shapepipe/modules/mask_ext_package/mask_ext.py create mode 100644 src/shapepipe/modules/mask_ext_runner.py create mode 100644 tests/module/test_mask_ext.py diff --git a/example/cfis/config_exp_MaExt.ini b/example/cfis/config_exp_MaExt.ini new file mode 100644 index 000000000..ecd416e13 --- /dev/null +++ b/example/cfis/config_exp_MaExt.ini @@ -0,0 +1,50 @@ +## ShapePipe configuration file for exposure-CCD external-mask (healsparse) +## rasterization. Reprojects the unified UNIONS healsparse mask through each +## single-exposure single-CCD WCS and sums in the instrument flag file, +## producing the pipeline_flag.fits artifact consumed downstream. + +[DEFAULT] +VERBOSE = True +RUN_NAME = run_sp_exp_MaExt +RUN_DATETIME = False + +[EXECUTION] +MODULE = mask_ext_runner +MODE = SMP + +[FILE] +LOG_NAME = log_sp_exp +RUN_LOG_NAME = log_run_sp +INPUT_DIR = $SP_RUN/output +OUTPUT_DIR = $SP_RUN/output + +[JOB] +SMP_BATCH_SIZE = 1 +TIMEOUT = 96:00:00 + +[MASK_EXT_RUNNER] + +# Parent module: single-exposure single-CCD images and instrument flags +INPUT_DIR = last:split_exp_runner + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Path of the external healsparse mask file (band-agnostic; r-band for shear) +MASK_PATH = $SP_CONFIG/mask_r.hsp + +# Healsparse bit value -> output flag value mapping +BIT_FLAG_MAP = 64:1 + +# Flag value for pixels outside the healsparse footprint (0 = unflagged) +OFF_MAP_FLAG = 0 + +# External instrument flag file: summed into the rasterized mask +USE_EXT_FLAG = True + +# HDU of the external instrument flag FITS file (optional, default 0) +HDU = 0 + +# File name prefix for the output flag files +PREFIX = pipeline diff --git a/example/cfis/config_tile_MaExt.ini b/example/cfis/config_tile_MaExt.ini new file mode 100644 index 000000000..410a6890d --- /dev/null +++ b/example/cfis/config_tile_MaExt.ini @@ -0,0 +1,51 @@ +## ShapePipe configuration file for tile external-mask (healsparse) rasterization +## Rasterizes the unified UNIONS healsparse mask onto each tile pixel grid, +## producing the pipeline_flag.fits artifact consumed downstream. + +[DEFAULT] +VERBOSE = True +RUN_NAME = run_sp_tile_MaExt +RUN_DATETIME = False + +[EXECUTION] +MODULE = mask_ext_runner +MODE = SMP + +[FILE] +LOG_NAME = log_sp_exp +RUN_LOG_NAME = log_run_sp +INPUT_DIR = $SP_RUN/output +OUTPUT_DIR = $SP_RUN/output + +[JOB] +SMP_BATCH_SIZE = 1 +TIMEOUT = 96:00:00 + +[MASK_EXT_RUNNER] + +# Input directory, tile image +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input file pattern(s): only the tile image is needed (WCS + pixel grid) +FILE_PATTERN = CFIS_image + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Path of the external healsparse mask file (band-agnostic; r-band for shear) +MASK_PATH = $SP_CONFIG/mask_r.hsp + +# Healsparse bit value -> output flag value mapping +BIT_FLAG_MAP = 64:1 + +# Flag value for pixels outside the healsparse footprint (0 = unflagged) +OFF_MAP_FLAG = 0 + +# No external instrument flag for tiles +USE_EXT_FLAG = False + +# File name prefix for the output flag files +PREFIX = pipeline diff --git a/src/shapepipe/modules/mask_ext_package/__init__.py b/src/shapepipe/modules/mask_ext_package/__init__.py new file mode 100644 index 000000000..4e2123986 --- /dev/null +++ b/src/shapepipe/modules/mask_ext_package/__init__.py @@ -0,0 +1,71 @@ +"""MASK EXT MODULE. + +This package contains the module for ``mask_ext``. + +:Author: Cail Daley + +:Parent module: ``split_exp_runner`` (exposures) or ``get_images_runner`` / + ``uncompress_fits_runner`` (tiles), or None + +:Input: Single image (tile or single-exposure single-CCD) and, optionally, an + external instrument flag file + +:Output: Per-image pixel flag file + +Description +=========== + +This module produces the ShapePipe per-image pixel flag file +``_flag.fits`` by rasterizing an **external healsparse mask** onto +the image pixel grid, rather than generating masks internally (the job of the +``mask`` module). It is the ShapePipe consumer of the unified UNIONS healsparse +mask products (PhotoPipe footprint + bright stars, MaxiMask, manual galaxy +masks), merged and stored as ``healsparse.HealSparseMap`` files. + +For each image the module: + +1. Reads the image header into an ``astropy.wcs.WCS``. +2. Evaluates the pixel grid to (RA, Dec) in row chunks (bounded memory). +3. Queries ``HealSparseMap.get_values_pos`` at every pixel centre. +4. Maps healsparse bit values to ShapePipe flag values through the + config-driven ``BIT_FLAG_MAP``, producing an ``int16`` flag image. +5. Optionally sums in an external instrument flag image (``USE_EXT_FLAG``), + matching the combination semantics of the ``mask`` module. +6. Writes ``_flag.fits`` carrying the image WCS in its header. + +Everything downstream (SExtractor ``IMAFLAGS_ISO``, setools star selection, +vignetmaker, ngmix) consumes this artifact unchanged; the module is a drop-in +replacement for ``mask`` at the flag-image contract. + +The same module serves tiles and exposure CCDs: only the input image (hence +WCS) differs. The healsparse file, its bit meanings, and all flag values live +**only in config** — the mask products evolve and are swapped at zero code cost. + +Module-specific config file entries +=================================== + +MASK_PATH : str + Path to the healsparse mask file (``.hsp``/``.fits``); band-agnostic +BIT_FLAG_MAP : str + Mapping from healsparse pixel bit value to output flag value, formatted as + ``":, :, ..."`` (e.g. ``"64:1, 2048:2"``). A pixel + carrying several bits receives the bitwise-OR of the mapped flag values. +OFF_MAP_FLAG : int, optional + Flag value assigned to pixels that fall outside the healsparse map + footprint (sentinel pixels); default is ``0``. Off-footprint pixels are + usually unobserved, so a non-zero value flags them. +USE_EXT_FLAG : bool, optional + If ``True``, sum an external instrument flag file (given as the second + input) into the rasterized mask; default is ``False``. Needed for + exposures, where saturation / bleeding / bad columns arrive with the data. +HDU : int, optional + HDU of the external instrument flag FITS file; default is ``0`` +PREFIX : str, optional + Prefix prepended to the output file name base ``flag``; default is ``""`` +CHUNK_SIZE : int, optional + Number of image rows evaluated per chunk; default is chosen from the image + size to bound memory (``1e8`` px for tiles, ``1e7`` px for exposure CCDs) + +""" + +__all__ = ["mask_ext"] diff --git a/src/shapepipe/modules/mask_ext_package/mask_ext.py b/src/shapepipe/modules/mask_ext_package/mask_ext.py new file mode 100644 index 000000000..4edc2a124 --- /dev/null +++ b/src/shapepipe/modules/mask_ext_package/mask_ext.py @@ -0,0 +1,319 @@ +"""MASK EXT. + +This module contains a class to rasterize an external healsparse mask onto an +image pixel grid, producing the ShapePipe per-image pixel flag file. + +:Author: Cail Daley + +""" + +import re + +import numpy as np +from astropy import wcs + +from shapepipe.pipeline import file_io + +# Per-chunk pixel budget: chunk the image in row bands so that no more than +# this many pixel coordinates are held / queried at once. A tile is ~1e8 px and +# an exposure CCD ~1e7 px; this bound keeps peak memory to a single band. +DEFAULT_PIXEL_BUDGET = 10_000_000 + + +class MaskExt(object): + """Mask Ext. + + Rasterize an external healsparse mask onto an image pixel grid and write + the resulting ShapePipe flag file. + + Parameters + ---------- + image_path : str + Path to the image whose pixel grid and WCS define the output + mask_path : str + Path to the external healsparse mask file + bit_flag_map : dict + Mapping from healsparse pixel bit value (int) to output flag value + (int); a pixel carrying several bits gets the bitwise-OR of the mapped + flags + image_num : str + File number string, inserted into the output file name + output_dir : str + Path to the output directory + w_log : logging.Logger + Log file + off_map_flag : int, optional + Flag value for pixels outside the healsparse footprint (sentinel + pixels); default is ``0`` + path_external_flag : str, optional + Path to an external instrument flag file to sum into the mask; default + is ``None`` (not used) + image_prefix : str, optional + Prefix prepended to the output file name base ``flag``; specify + ``'none'`` or ``''`` for no prefix, default is ``''`` + outname_base : str, optional + Output file name base, default is ``flag`` + chunk_size : int, optional + Number of image rows evaluated per chunk; default is derived from the + image width and :data:`DEFAULT_PIXEL_BUDGET` + hdu : int, optional + HDU of the external instrument flag FITS file; default is ``0`` + + """ + + def __init__( + self, + image_path, + mask_path, + bit_flag_map, + image_num, + output_dir, + w_log, + off_map_flag=0, + path_external_flag=None, + image_prefix="", + outname_base="flag", + chunk_size=None, + hdu=0, + ): + self._image_path = image_path + self._mask_path = mask_path + self._bit_flag_map = bit_flag_map + self._img_number = image_num + self._output_dir = output_dir + self._w_log = w_log + self._off_map_flag = int(off_map_flag) + self._path_external_flag = path_external_flag + + if (image_prefix.lower() != "none") and (image_prefix != ""): + self._img_prefix = f"{image_prefix}_" + else: + self._img_prefix = "" + + self._outname_base = outname_base + self._chunk_size = chunk_size + self._hdu = hdu + + self._set_image_coordinates() + + @staticmethod + def parse_bit_flag_map(map_string): + """Parse Bit Flag Map. + + Parse the ``BIT_FLAG_MAP`` config string into a dictionary. + + Parameters + ---------- + map_string : str + Mapping formatted as ``":, :, ..."``, e.g. + ``"64:1, 2048:2"`` + + Returns + ------- + dict + Mapping from healsparse bit value (int) to output flag value (int) + + Raises + ------ + ValueError + If an entry is not of the form ``:`` + + """ + bit_flag_map = {} + for entry in map_string.split(","): + entry = entry.strip() + if not entry: + continue + if not re.fullmatch(r"\d+\s*:\s*\d+", entry): + raise ValueError( + f"Invalid BIT_FLAG_MAP entry '{entry}'; expected " + + "':'" + ) + bit, flag = (int(part) for part in entry.split(":")) + bit_flag_map[bit] = flag + if not bit_flag_map: + raise ValueError("BIT_FLAG_MAP is empty") + return bit_flag_map + + def _set_image_coordinates(self): + """Set Image Coordinates. + + Read the image header into a WCS and record the image shape, mirroring + ``Mask._set_image_coordinates``. + + """ + img = file_io.FITSCatalogue(self._image_path, hdu_no=0) + img.open() + self._header = img.get_header() + # get_data().shape is (n_y, n_x) + self._img_shape = img.get_data().shape + img.close() + del img + + self._wcs = wcs.WCS(self._header) + + def _default_chunk_size(self): + """Default Chunk Size. + + Rows per chunk such that one band holds at most + :data:`DEFAULT_PIXEL_BUDGET` pixels. + + Returns + ------- + int + Number of rows per chunk (at least 1) + + """ + n_x = self._img_shape[1] + return max(1, DEFAULT_PIXEL_BUDGET // n_x) + + def _map_bits_to_flags(self, bit_values, off_map): + """Map Bits to Flags. + + Translate healsparse bit values to output flag values through the + bit→flag mapping, OR-combining every matching bit, and assign the + off-map flag to sentinel pixels. + + Parameters + ---------- + bit_values : numpy.ndarray + Healsparse values queried at the pixel centres + off_map : numpy.ndarray + Boolean mask, ``True`` where the pixel lies outside the footprint + + Returns + ------- + numpy.ndarray + Output flag values (``int16``), same shape as ``bit_values`` + + """ + flags = np.zeros(bit_values.shape, dtype=np.int16) + for bit, flag in self._bit_flag_map.items(): + flags[(bit_values & bit) != 0] |= np.int16(flag) + if self._off_map_flag != 0: + flags[off_map] = np.int16(self._off_map_flag) + return flags + + def rasterize(self): + """Rasterize. + + Evaluate the pixel grid to (RA, Dec) in row chunks, query the + healsparse mask, and build the ``int16`` flag image. + + Returns + ------- + numpy.ndarray + The rasterized flag image, shape ``(n_y, n_x)``, dtype ``int16`` + + """ + # Lazy import: healsparse is an optional heavy dependency, imported at + # use rather than module load. + import healsparse + + hmap = healsparse.HealSparseMap.read(self._mask_path) + sentinel = hmap.sentinel + + n_y, n_x = self._img_shape + chunk_size = self._chunk_size or self._default_chunk_size() + + flag_image = np.zeros((n_y, n_x), dtype=np.int16) + # Column indices are shared across every row band. + x = np.arange(n_x) + + for y0 in range(0, n_y, chunk_size): + y1 = min(y0 + chunk_size, n_y) + yy, xx = np.meshgrid(np.arange(y0, y1), x, indexing="ij") + # WCS uses 0-based pixel coordinates here (origin=0). + ra, dec = self._wcs.all_pix2world(xx.ravel(), yy.ravel(), 0) + # Normalize RA into [0, 360) so the wrap at 0/360 is handled; + # healsparse expects lon in that range with lonlat=True. + ra = np.mod(ra, 360.0) + + bit_values = hmap.get_values_pos(ra, dec, lonlat=True) + off_map = bit_values == sentinel + + band = self._map_bits_to_flags(bit_values, off_map) + flag_image[y0:y1, :] = band.reshape(y1 - y0, n_x) + + return flag_image + + def _combine_external_flag(self, flag_image): + """Combine External Flag. + + Sum an external instrument flag image into the rasterized mask, + matching ``Mask._build_final_mask`` semantics (element-wise sum of the + two integer flag images). + + Parameters + ---------- + flag_image : numpy.ndarray + The rasterized flag image + + Returns + ------- + numpy.ndarray + The combined flag image (``int16``) + + """ + external_flag = file_io.FITSCatalogue( + self._path_external_flag, + hdu_no=self._hdu, + ) + external_flag.open() + ext_flag = external_flag.get_data()[:, :] + external_flag.close() + + return (flag_image + ext_flag).astype(np.int16, copy=False) + + def _output_path(self): + """Output Path. + + Full path of the output flag file, matching the ``mask`` module naming + (``_flag.fits``). + + Returns + ------- + str + Output file path + + """ + name = ( + f"{self._img_prefix}{self._outname_base}" + + f"{self._img_number}.fits" + ) + return f"{self._output_dir}/{name}" + + def make_mask(self): + """Make Mask. + + Rasterize the healsparse mask, optionally combine the external + instrument flag, and write the flag file carrying the image WCS. + + Returns + ------- + str + Path to the written flag file + + """ + flag_image = self.rasterize() + + if self._path_external_flag is not None: + flag_image = self._combine_external_flag(flag_image) + + output_path = self._output_path() + out = file_io.FITSCatalogue( + output_path, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + hdu_no=0, + ) + out.save_as_fits( + data=flag_image, + image=True, + image_header=self._wcs.to_header(), + ) + + self._w_log.info( + f"Wrote healsparse-derived flag file {output_path}" + ) + + return output_path diff --git a/src/shapepipe/modules/mask_ext_runner.py b/src/shapepipe/modules/mask_ext_runner.py new file mode 100644 index 000000000..a43265d1b --- /dev/null +++ b/src/shapepipe/modules/mask_ext_runner.py @@ -0,0 +1,117 @@ +"""MASK EXT RUNNER. + +Module runner for ``mask_ext``. + +:Author: Cail Daley + +""" + +from shapepipe.modules.mask_ext_package.mask_ext import MaskExt +from shapepipe.modules.module_decorator import module_runner + + +@module_runner( + version="1.0", + file_pattern=["image", "flag"], + file_ext=[".fits", ".fits"], + depends=["numpy", "astropy", "healsparse"], + numbering_scheme="_0", +) +def mask_ext_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Mask Ext Runner. + + Rasterize an external healsparse mask onto the input image pixel grid and + write the ShapePipe flag file. + + Notes + ----- + Only the image is strictly required: it supplies the pixel grid and WCS + onto which the healsparse mask is rasterized. Unlike the ``mask`` module, + no weight file is used — the healsparse mask already encodes the footprint, + and missing-data handling is not this module's concern. A second input, an + external instrument flag file, is consumed only when ``USE_EXT_FLAG`` is + ``True`` (needed for exposures, where saturation / bleeding / bad columns + arrive with the data). + + """ + n_inputs = len(input_file_list) + use_ext_flag = config.getboolean(module_config_sec, "USE_EXT_FLAG") if ( + config.has_option(module_config_sec, "USE_EXT_FLAG") + ) else False + + if use_ext_flag: + if n_inputs != 2: + raise ValueError( + f"Found {n_inputs} inputs but USE_EXT_FLAG is True, which " + + 'expects "image" and "flag" in the MASK_EXT_RUNNER section ' + + "of the config file." + ) + image_path = input_file_list[0] + ext_flag_name = input_file_list[1] + else: + if n_inputs != 1: + raise ValueError( + f"Found {n_inputs} inputs but USE_EXT_FLAG is False, which " + + 'expects only "image" in the MASK_EXT_RUNNER section of the ' + + "config file." + ) + image_path = input_file_list[0] + ext_flag_name = None + + # Path to the healsparse mask file + mask_path = config.getexpanded(module_config_sec, "MASK_PATH") + + # Bit -> flag mapping + bit_flag_map = MaskExt.parse_bit_flag_map( + config.get(module_config_sec, "BIT_FLAG_MAP") + ) + + # Flag value for pixels outside the healsparse footprint + if config.has_option(module_config_sec, "OFF_MAP_FLAG"): + off_map_flag = config.getint(module_config_sec, "OFF_MAP_FLAG") + else: + off_map_flag = 0 + + # HDU of the external instrument flag file + if config.has_option(module_config_sec, "HDU"): + hdu = config.getint(module_config_sec, "HDU") + else: + hdu = 0 + + # Output file name prefix + if config.has_option(module_config_sec, "PREFIX"): + prefix = config.get(module_config_sec, "PREFIX") + else: + prefix = "" + + # Rows per chunk (optional; default derived from image size) + if config.has_option(module_config_sec, "CHUNK_SIZE"): + chunk_size = config.getint(module_config_sec, "CHUNK_SIZE") + else: + chunk_size = None + + mask_inst = MaskExt( + image_path, + mask_path, + bit_flag_map, + file_number_string, + run_dirs["output"], + w_log, + off_map_flag=off_map_flag, + path_external_flag=ext_flag_name, + image_prefix=prefix.replace(" ", ""), + outname_base="flag", + chunk_size=chunk_size, + hdu=hdu, + ) + + mask_inst.make_mask() + + return None, None diff --git a/tests/module/test_mask_ext.py b/tests/module/test_mask_ext.py new file mode 100644 index 000000000..f1843f2c3 --- /dev/null +++ b/tests/module/test_mask_ext.py @@ -0,0 +1,230 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MASK_EXT. + +Drives ``MaskExt`` against a synthetic healsparse map and a synthetic TAN WCS +to lock in the rasterization contract of ``mask_ext_runner``: the config-driven +bit->flag mapping (with bitwise-OR of multiple bits), chunk-seam independence, +RA wrap across 0/360, off-footprint (sentinel) handling, external-flag summing, +``int16`` output dtype, and WCS round-trip in the written FITS. + +The synthetic map is a high-resolution healsparse map covering only a tiny +patch on the sky; the synthetic WCS points an image at that patch so a subset +of pixels land on masked healpix cells and the rest fall off the footprint. +""" + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits +from astropy.wcs import WCS + +healsparse = pytest.importorskip("healsparse") +hpgeom = pytest.importorskip("hpgeom") + +from shapepipe.modules.mask_ext_package.mask_ext import MaskExt + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +# Sky patch the synthetic image and mask share. +CRVAL1 = 150.0 +CRVAL2 = 2.3 +NSIDE_SPARSE = 131072 # ~1.6 arcsec, matching the real UNIONS masks +NSIDE_COVERAGE = 32 +PIXSCALE_DEG = 0.187 / 3600.0 # UNIONS ~0.187"/px + + +def _make_wcs(naxis1, naxis2, crval1=CRVAL1, crval2=CRVAL2): + """Synthetic TAN WCS header centred on the shared patch.""" + w = WCS(naxis=2) + w.wcs.ctype = ["RA---TAN", "DEC--TAN"] + w.wcs.crval = [crval1, crval2] + w.wcs.crpix = [naxis1 / 2 + 0.5, naxis2 / 2 + 0.5] + w.wcs.cd = [[-PIXSCALE_DEG, 0.0], [0.0, PIXSCALE_DEG]] + return w + + +def _write_image(path, wcs_obj, naxis1, naxis2): + """Write a zero-valued image carrying the given WCS (defines pixel grid).""" + data = np.zeros((naxis2, naxis1), dtype=np.float32) + hdu = fits.PrimaryHDU(data) + # NAXIS* are set from the data shape; merge only the WCS keywords. + hdu.header.update(wcs_obj.to_header()) + hdu.writeto(path, overwrite=True) + + +def _make_map(masked_ra, masked_dec, bits, sentinel=0): + """Healsparse int32 map with the given (ra, dec) cells set to ``bits``.""" + hmap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.int32, sentinel=sentinel + ) + pix = hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec) + bits = np.asarray(bits, dtype=np.int32) + # Several fine image pixels can share one healpix cell; keep the first bit + # value per unique cell (replace requires unique pixels). + pix, first = np.unique(pix, return_index=True) + hmap[pix] = bits[first] + return hmap + + +def _run(tmp_path, image_wcs, naxis1, naxis2, hmap, bit_flag_map, + off_map_flag=0, chunk_size=None, path_external_flag=None): + """Instantiate MaskExt on written synthetic inputs and rasterize/write.""" + image_path = str(tmp_path / "image.fits") + mask_path = str(tmp_path / "mask.hsp") + _write_image(image_path, image_wcs, naxis1, naxis2) + hmap.write(mask_path, clobber=True) + + inst = MaskExt( + image_path, + mask_path, + bit_flag_map, + image_num="-000-000", + output_dir=str(tmp_path), + w_log=_NullLogger(), + off_map_flag=off_map_flag, + path_external_flag=path_external_flag, + image_prefix="pipeline", + chunk_size=chunk_size, + ) + return inst + + +def test_parse_bit_flag_map(): + """String config parses into an int->int dict; malformed entries raise.""" + assert MaskExt.parse_bit_flag_map("64:1, 2048:2") == {64: 1, 2048: 2} + assert MaskExt.parse_bit_flag_map(" 64 : 1 ") == {64: 1} + with pytest.raises(ValueError): + MaskExt.parse_bit_flag_map("64") + with pytest.raises(ValueError): + MaskExt.parse_bit_flag_map("") + + +def test_bit_flag_mapping_and_off_map(tmp_path): + """Masked cells map their bit; unmasked (off-footprint) get off_map_flag. + + The image centre pixel lands on a cell carrying bit 64 -> flag 1; the map + covers only that centre cell, so every other pixel is off-footprint. + """ + naxis1 = naxis2 = 16 + w = _make_wcs(naxis1, naxis2) + # RA/Dec at the central pixel (0-based grid, origin=0). + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, off_map_flag=8) + flags = inst.rasterize() + + assert flags.dtype == np.int16 + assert flags.shape == (naxis2, naxis1) + # The centre cell is flagged 1; the rest of the image is off the footprint. + n_flagged_1 = np.sum(flags == 1) + assert n_flagged_1 >= 1 + assert np.all(flags[flags != 1] == 8) + assert np.sum(flags == 8) == flags.size - n_flagged_1 + + +def test_multi_bit_or(tmp_path): + """A cell carrying two bits gets the bitwise-OR of the mapped flags.""" + naxis1 = naxis2 = 8 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + # bit values 64 and 2048 set together on the centre cell. + hmap = _make_map([float(ra_c)], [float(dec_c)], [64 | 2048]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1, 2048: 2}) + flags = inst.rasterize() + # 1 | 2 == 3 on the masked cell. + assert 3 in np.unique(flags) + assert np.all(np.isin(np.unique(flags), [0, 3])) + + +def test_chunk_seam_independence(tmp_path): + """Result is independent of chunk size (no seam artifacts).""" + naxis1 = naxis2 = 40 + w = _make_wcs(naxis1, naxis2) + # Mask a band of cells spanning several image rows. + ys = np.arange(0, naxis2) + xs = np.full_like(ys, naxis1 // 2) + ra, dec = w.all_pix2world(xs, ys, 0) + hmap = _make_map(ra.astype(float), dec.astype(float), [64] * len(ys)) + + inst_full = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + full = inst_full.rasterize() + + for cs in (1, 3, 7, 40): + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, chunk_size=cs) + npt.assert_array_equal(inst.rasterize(), full) + # Sanity: some pixels actually got flagged. + assert np.any(full == 1) + + +def test_ra_wrap(tmp_path): + """RA-wrap near 0/360: an image centred on RA~0 rasterizes correctly.""" + naxis1 = naxis2 = 16 + w = _make_wcs(naxis1, naxis2, crval1=0.0, crval2=2.3) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + # Straddling pixels produce RA both just below 360 and just above 0; the + # map cell is queried by its wrapped-into-[0,360) coordinate. + hmap = _make_map([float(np.mod(ra_c, 360.0))], [float(dec_c)], [64]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + flags = inst.rasterize() + # No crash, correct dtype, and the centre is flagged despite the wrap. + assert flags.dtype == np.int16 + assert np.any(flags == 1) + + +def test_external_flag_summing(tmp_path): + """External instrument flag image is summed into the rasterized mask.""" + naxis1 = naxis2 = 8 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) + + # External flag: a constant field of 16 (e.g. a saturation bit). + ext_path = str(tmp_path / "ext_flag.fits") + ext_data = np.full((naxis2, naxis1), 16, dtype=np.int16) + fits.PrimaryHDU(ext_data).writeto(ext_path, overwrite=True) + + inst = _run( + tmp_path, w, naxis1, naxis2, hmap, {64: 1}, + path_external_flag=ext_path, + ) + out_path = inst.make_mask() + + with fits.open(out_path) as hdul: + written = hdul[0].data + + # FITS stores big-endian; still a 2-byte signed int (int16). + assert written.dtype.kind == "i" and written.dtype.itemsize == 2 + # Every pixel gets +16 from the external flag; the centre cell also +1. + assert np.all(written >= 16) + assert 17 in np.unique(written) + + +def test_write_wcs_roundtrip(tmp_path): + """Written FITS carries the image WCS; pix2world round-trips.""" + naxis1 = naxis2 = 12 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + out_path = inst.make_mask() + + with fits.open(out_path) as hdul: + assert hdul[0].data.dtype.kind == "i" + assert hdul[0].data.dtype.itemsize == 2 + w_out = WCS(hdul[0].header) + + ra_in, dec_in = w.all_pix2world(cx, cy, 0) + ra_out, dec_out = w_out.all_pix2world(cx, cy, 0) + npt.assert_allclose([ra_in, dec_in], [ra_out, dec_out], rtol=0, atol=1e-9) From 6baa474494a92f74d295078b6016f2e54cd1b205 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 17:46:06 +0200 Subject: [PATCH 03/17] feat(make_cat): per-band MASK_ columns from external healsparse maps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optional MASK_EXT_PATHS (band:path pairs, env-expanded) in the make_cat section: each band's healsparse map is queried at object world positions (XWIN_WORLD/YWIN_WORLD) and added as a MASK_ column — the ShapePipe end of UNIONS-WL/spherex#38. Strict no-op when unset; off-map objects carry the map sentinel (-1 for integer maps). 3 unit tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CfRuCQa2UHo44yp2MZDsJX --- .../modules/make_cat_package/make_cat.py | 62 +++++++++ src/shapepipe/modules/make_cat_runner.py | 10 ++ tests/module/test_make_cat_mask_ext.py | 129 ++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 tests/module/test_make_cat_mask_ext.py diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 24e72693a..d6c38134d 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -209,6 +209,68 @@ def save_sm_data( return n_obj +def parse_mask_ext_paths(paths_str): + """Parse Mask Ext Paths. + + Parse the ``MASK_EXT_PATHS`` config value into a ``band -> path`` mapping. + + Parameters + ---------- + paths_str : str + Comma-separated ``band:path`` pairs, e.g. + ``u:/path/mask_u.hsp, g:/path/mask_g.hsp`` + + Returns + ------- + dict + Mapping from band name to healsparse map path + + """ + band_paths = {} + for pair in paths_str.split(","): + band, path = pair.split(":", 1) + band_paths[band.strip()] = path.strip() + + return band_paths + + +def save_mask_ext_data(final_cat_file, band_paths, w_log): + """Save External Mask Data. + + Query per-band external healsparse masks at each object's world position + and write one ``MASK_`` column per band into the final catalogue. + Object positions are read from the SExtractor windowed world coordinates + (``XWIN_WORLD`` = RA, ``YWIN_WORLD`` = Dec, both in degrees) carried in the + ``RESULTS`` extension. Objects falling outside a map's coverage receive + that map's sentinel value (``healsparse.HealSparseMap.get_values_pos`` + returns the map's sentinel — ``-1`` for integer maps — verbatim), which is + the documented off-map flag. + + Parameters + ---------- + final_cat_file : file_io.FITSCatalogue + Final catalogue + band_paths : dict + Mapping from band name to healsparse map path + w_log : logging.Logger + Logging instance + + """ + import healsparse + + final_cat_file.open() + ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"]) + dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"]) + + for band, path in band_paths.items(): + w_log.info(f"Query external mask for band {band}: {path}") + mask_map = healsparse.HealSparseMap.read(path) + values = mask_map.get_values_pos(ra, dec, lonlat=True) + final_cat_file.add_col(f"MASK_{band}", np.asarray(values)) + + final_cat_file.close() + + class SaveCatalogue: """Save Catalogue. diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 307dc2ffe..176341757 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -137,4 +137,14 @@ def make_cat_runner( if save_psf: err_msg = sc_inst.process("psf", galaxy_psf_path) + # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38): + # add one MASK_ column per band, queried at each object's world + # position. Absent config is a strict no-op. + if config.has_option(module_config_sec, "MASK_EXT_PATHS"): + band_paths = make_cat.parse_mask_ext_paths( + config.getexpanded(module_config_sec, "MASK_EXT_PATHS") + ) + w_log.info("Save external mask data") + make_cat.save_mask_ext_data(final_cat_file, band_paths, w_log) + return None, None diff --git a/tests/module/test_make_cat_mask_ext.py b/tests/module/test_make_cat_mask_ext.py new file mode 100644 index 000000000..91d0f5154 --- /dev/null +++ b/tests/module/test_make_cat_mask_ext.py @@ -0,0 +1,129 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MAKE_CAT external-mask columns. + +Exercises the optional per-band external healsparse mask lookup added to +``make_cat`` (PR #847 §4, the ShapePipe end of UNIONS-WL/spherex#38). A small +synthetic ``final_cat`` FITS carrying known ``XWIN_WORLD`` / ``YWIN_WORLD`` +object positions is queried against synthetic healsparse maps of known value, +locking in: (1) the ``MASK_`` column name and per-object values, (2) the +off-map sentinel (``-1`` for integer maps) written verbatim for objects outside +coverage, (3) multi-band handling, and (4) that absent config leaves the +catalogue untouched. +""" + +import numpy as np +import numpy.testing as npt +import pytest + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.make_cat_package import make_cat +from shapepipe.pipeline import file_io + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Object world positions (RA, Dec in degrees). The last object sits far from +# the map coverage so it exercises the off-map sentinel path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +def _make_map(value, dtype=np.int16, sentinel=-1): + """Build a healsparse map covering the first three RA/Dec positions. + + All covered pixels carry ``value``; everything else reads the sentinel. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=sentinel + ) + smap.update_values_pos( + RA[:3], DEC[:3], np.full(3, value, dtype=dtype), lonlat=True + ) + return smap + + +def _write_final_cat(path): + """Write a synthetic final_cat FITS with a RESULTS ext of known positions.""" + data = np.empty( + len(RA), + dtype=[ + ("NUMBER", "i4"), + ("XWIN_WORLD", "f8"), + ("YWIN_WORLD", "f8"), + ], + ) + data["NUMBER"] = np.arange(len(RA)) + data["XWIN_WORLD"] = RA + data["YWIN_WORLD"] = DEC + + cat = file_io.FITSCatalogue( + str(path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + cat.save_as_fits(data, ext_name="RESULTS") + return cat + + +def test_parse_mask_ext_paths(): + """band:path pairs parse into a stripped mapping, whitespace-tolerant.""" + parsed = make_cat.parse_mask_ext_paths( + "u:/a/mask_u.hsp, g:/b/mask_g.hsp,r:/c/mask_r.hsp" + ) + assert parsed == { + "u": "/a/mask_u.hsp", + "g": "/b/mask_g.hsp", + "r": "/c/mask_r.hsp", + } + + +def test_mask_ext_columns(tmp_path): + """Per-band columns carry the map value on-map and the sentinel off-map.""" + u_map = _make_map(16) + g_map = _make_map(32) + u_path = tmp_path / "mask_u.hsp" + g_path = tmp_path / "mask_g.hsp" + u_map.write(str(u_path)) + g_map.write(str(g_path)) + + cat_path = tmp_path / "final_cat-000.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue( + str(cat_path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + make_cat.save_mask_ext_data( + cat, + {"u": str(u_path), "g": str(g_path)}, + _NullLogger(), + ) + + cat.open() + data = cat.get_data() + # On-map objects (first three) carry the map value; the off-map object + # (last) carries the map's -1 sentinel. + npt.assert_array_equal(data["MASK_u"], [16, 16, 16, -1]) + npt.assert_array_equal(data["MASK_g"], [32, 32, 32, -1]) + # Integer dtype preserved from the map. + assert np.issubdtype(data["MASK_u"].dtype, np.integer) + cat.close() + + +def test_mask_ext_absent_is_noop(tmp_path): + """Not calling the lookup leaves the catalogue columns unchanged.""" + cat_path = tmp_path / "final_cat-001.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue(str(cat_path)) + cat.open() + cols = set(cat.get_data().dtype.names) + cat.close() + + assert cols == {"NUMBER", "XWIN_WORLD", "YWIN_WORLD"} + assert not any(name.startswith("MASK_") for name in cols) From ba20d53961dfb5ee179a8aae2bcf67f99ab06d79 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 18:32:24 +0200 Subject: [PATCH 04/17] feat(mask_ext): fail fast on boolean masks with non-1 bits The real 2025 r-band UNIONS mask (mask_r_nside131072.hsp) is a boolean healsparse map (True = masked), not an integer bit-flag map. With a boolean map, any BIT_FLAG_MAP bit other than 1 silently selects nothing (True & 64 == 0), producing an all-clean flag image. Raise instead, and document the two mask flavours in the example configs. Found by the real-data smoke: rasterizing the candide mask copy onto the CFIS.233.293 tile grid. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_013btLTHCgmiiggZmxJ4hM3n --- example/cfis/config_exp_MaExt.ini | 4 +- example/cfis/config_tile_MaExt.ini | 4 +- .../modules/mask_ext_package/mask_ext.py | 14 ++++++ tests/module/test_mask_ext.py | 44 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/example/cfis/config_exp_MaExt.ini b/example/cfis/config_exp_MaExt.ini index ecd416e13..ccb527dd1 100644 --- a/example/cfis/config_exp_MaExt.ini +++ b/example/cfis/config_exp_MaExt.ini @@ -34,7 +34,9 @@ NUMBERING_SCHEME = -0000000-0 # Path of the external healsparse mask file (band-agnostic; r-band for shear) MASK_PATH = $SP_CONFIG/mask_r.hsp -# Healsparse bit value -> output flag value mapping +# Healsparse bit value -> output flag value mapping. +# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks +# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. BIT_FLAG_MAP = 64:1 # Flag value for pixels outside the healsparse footprint (0 = unflagged) diff --git a/example/cfis/config_tile_MaExt.ini b/example/cfis/config_tile_MaExt.ini index 410a6890d..fce00da1d 100644 --- a/example/cfis/config_tile_MaExt.ini +++ b/example/cfis/config_tile_MaExt.ini @@ -38,7 +38,9 @@ FILE_EXT = .fits # Path of the external healsparse mask file (band-agnostic; r-band for shear) MASK_PATH = $SP_CONFIG/mask_r.hsp -# Healsparse bit value -> output flag value mapping +# Healsparse bit value -> output flag value mapping. +# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks +# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. BIT_FLAG_MAP = 64:1 # Flag value for pixels outside the healsparse footprint (0 = unflagged) diff --git a/src/shapepipe/modules/mask_ext_package/mask_ext.py b/src/shapepipe/modules/mask_ext_package/mask_ext.py index 4edc2a124..c24aaabcd 100644 --- a/src/shapepipe/modules/mask_ext_package/mask_ext.py +++ b/src/shapepipe/modules/mask_ext_package/mask_ext.py @@ -213,6 +213,20 @@ def rasterize(self): hmap = healsparse.HealSparseMap.read(self._mask_path) sentinel = hmap.sentinel + # Mask products come in two flavours: integer bit-flag maps (per-band + # bits, e.g. 64 = r) and boolean maps (True = masked, e.g. the + # candide copy of the 2025 r-band mask). For a boolean map the only + # meaningful bit is 1 (True); any other bit silently selects nothing + # (``True & 64 == 0`` -> an all-clean flag image), so fail loudly. + if hmap.dtype == np.bool_: + bad_bits = [bit for bit in self._bit_flag_map if bit != 1] + if bad_bits: + raise ValueError( + f"Mask {self._mask_path} is a boolean healsparse map; " + + f"BIT_FLAG_MAP bits {bad_bits} would never match " + + "(use '1:' for boolean masks)" + ) + n_y, n_x = self._img_shape chunk_size = self._chunk_size or self._default_chunk_size() diff --git a/tests/module/test_mask_ext.py b/tests/module/test_mask_ext.py index f1843f2c3..6958c9e02 100644 --- a/tests/module/test_mask_ext.py +++ b/tests/module/test_mask_ext.py @@ -228,3 +228,47 @@ def test_write_wcs_roundtrip(tmp_path): ra_in, dec_in = w.all_pix2world(cx, cy, 0) ra_out, dec_out = w_out.all_pix2world(cx, cy, 0) npt.assert_allclose([ra_in, dec_in], [ra_out, dec_out], rtol=0, atol=1e-9) + + +def _make_bool_map(masked_ra, masked_dec): + """Healsparse boolean map (True = masked) at the given (ra, dec) cells.""" + hmap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.bool_, sentinel=False + ) + pix = np.unique(hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec)) + hmap[pix] = np.ones(len(pix), dtype=np.bool_) + return hmap + + +def test_bool_map_flag_1(tmp_path): + """A boolean mask (True = masked) rasterizes through BIT_FLAG_MAP 1:1. + + This is the flavour of the real 2025 r-band UNIONS mask + (``mask_r_nside131072.hsp``): dtype bool, sentinel False, valid pixels + only where masked. + """ + naxis1 = naxis2 = 16 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {1: 1}) + flags = inst.rasterize() + + assert flags.dtype == np.int16 + assert np.sum(flags == 1) >= 1 + assert np.all(np.isin(np.unique(flags), [0, 1])) + + +def test_bool_map_wrong_bits_raise(tmp_path): + """Boolean mask + bits other than 1 would silently select nothing: raise.""" + naxis1 = naxis2 = 8 + w = _make_wcs(naxis1, naxis2) + cx, cy = naxis1 // 2, naxis2 // 2 + ra_c, dec_c = w.all_pix2world(cx, cy, 0) + hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) + + inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) + with pytest.raises(ValueError, match="boolean healsparse map"): + inst.rasterize() From f554e2b51e417596472ae50091b4121a39bcb730 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 18:29:02 -0400 Subject: [PATCH 05/17] Add MASK_EXT_PATHS example to make_cat config The catalogue-level mask query path (parse_mask_ext_paths, save_mask_ext_data) added in this PR had no shipped example. Document MASK_EXT_PATHS in config_make_cat_psfex.ini, the tile-level MAKE_CAT_RUNNER config, matching the design language used for the rasterizer's BIT_FLAG_MAP in config_tile_MaExt.ini. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KQnUyBXB85PdFF4xAKC6WC --- example/cfis/config_make_cat_psfex.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/example/cfis/config_make_cat_psfex.ini b/example/cfis/config_make_cat_psfex.ini index a7407d990..0690a69b9 100644 --- a/example/cfis/config_make_cat_psfex.ini +++ b/example/cfis/config_make_cat_psfex.ini @@ -75,3 +75,9 @@ SM_STAR_THRESH = 0.003 SM_GAL_THRESH = 0.01 SHAPE_MEASUREMENT_TYPE = ngmix + +# Optional per-band external healsparse mask lookup (band:path pairs, +# comma-separated). All bits of each map are queried at each object's +# (RA, Dec) and written verbatim to a MASK_ column; no filtering +# is applied here. Absent this key, the step is a no-op. +; MASK_EXT_PATHS = r:$SP_CONFIG/mask_r.hsp, u:$SP_CONFIG/mask_u.hsp From eb93838adad500adcfa59872b381b52b94f143de Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:45:26 -0400 Subject: [PATCH 06/17] refactor(mask): delete internal mask generation and its star catalogues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShapePipe stops making masks (#846, part of #845). Gone in one piece: the `mask` module (GSC 2.3 Vizier queries, Messier/NGC region files, the WeightWatcher .ww default and the halo/star .reg templates), the `mask_ext` rasterizer that briefly stood in for it, and every config that ran either — the *.mask / *.mask_simu mask configs, config_{exp,tile}_Ma_onthefly.ini, config_{exp,tile}_MaExt.ini, the defunct MaMa and Sx_exp_* variants, and workflow/config/cfis/config_exp_Ma.ini with its three dangling symlinks. The star-catalogue staging the mask module needed goes with it: the `star_catalogue` / `exp_star_cat` Snakemake halves (workflow/scripts/star_cats.py), scripts/python/create_star_cat.py and collate_star_cat.py, and shapepipe.utilities.vizier. Two utilities become dead in the same stroke and are deleted rather than left orphaned: `focal_plane` (only create_star_cat and star_cats.py called focal_plane_disc) and `utilities.file_io` (write_atomic had exactly those two callers; it is prose-cited from tile.smk, fixed there). Nothing replaces them with code. The sky-fixed masks are healsparse maps and are queried per object — MASK_ columns in make_cat, FLAG_EXT in the new mask_query module — so what used to be a pipeline stage is now a config entry. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_exp_MaExt.ini | 52 - example/cfis/config_exp_Ma_onthefly.ini | 79 - example/cfis/config_onthefly.mask | 86 -- example/cfis/config_save.mask | 86 -- example/cfis/config_tile_MaExt.ini | 53 - example/cfis/config_tile_Ma_onthefly.ini | 82 -- example/cfis/config_tile_onthefly.mask | 89 -- example/cfis/config_tile_save.mask | 89 -- example/cfis/defunct/config_MaMa_onthefly.ini | 105 -- example/cfis/defunct/config_MaMa_save.ini | 109 -- .../cfis/defunct/config_tile_Sx_exp_mccd.ini | 274 ---- .../cfis/defunct/config_tile_Sx_exp_psfex.ini | 248 ---- .../mask_default/MEGAPRIME_star_i_13.8.reg | 24 - example/cfis/mask_default/Messier_catalog.npy | Bin 4209 -> 0 bytes .../mask_default/Messier_catalog_updated.fits | Bin 8640 -> 0 bytes example/cfis/mask_default/default.ww | 40 - example/cfis/mask_default/halo_mask.reg | 50 - example/cfis/mask_default/ngc_cat.fits | Bin 201600 -> 0 bytes .../config_exp_Ma_onthefly.ini | 76 - .../cfis_image_sims/config_onthefly.mask_simu | 86 -- .../config_tile_Ma_onthefly.ini | 82 -- .../config_tile_onthefly.mask_simu | 89 -- scripts/python/collate_star_cat.py | 895 ------------ scripts/python/create_star_cat.py | 104 -- .../modules/mask_ext_package/__init__.py | 71 - .../modules/mask_ext_package/mask_ext.py | 333 ----- src/shapepipe/modules/mask_ext_runner.py | 117 -- .../modules/mask_package/__init__.py | 185 --- src/shapepipe/modules/mask_package/mask.py | 1271 ----------------- src/shapepipe/modules/mask_runner.py | 120 -- src/shapepipe/utilities/file_io.py | 45 - src/shapepipe/utilities/focal_plane.py | 92 -- src/shapepipe/utilities/vizier.py | 101 -- tests/module/test_collate_star_cat.py | 70 - tests/module/test_mask_ext.py | 274 ---- workflow/config/cfis/config_exp_Ma.ini | 86 -- workflow/config/cfis/config_onthefly.mask | 1 - .../config/cfis/config_tile_onthefly.mask | 1 - workflow/config/cfis/mask_default | 1 - workflow/scripts/star_cats.py | 301 ---- 40 files changed, 5867 deletions(-) delete mode 100644 example/cfis/config_exp_MaExt.ini delete mode 100644 example/cfis/config_exp_Ma_onthefly.ini delete mode 100644 example/cfis/config_onthefly.mask delete mode 100644 example/cfis/config_save.mask delete mode 100644 example/cfis/config_tile_MaExt.ini delete mode 100644 example/cfis/config_tile_Ma_onthefly.ini delete mode 100644 example/cfis/config_tile_onthefly.mask delete mode 100644 example/cfis/config_tile_save.mask delete mode 100644 example/cfis/defunct/config_MaMa_onthefly.ini delete mode 100644 example/cfis/defunct/config_MaMa_save.ini delete mode 100644 example/cfis/defunct/config_tile_Sx_exp_mccd.ini delete mode 100644 example/cfis/defunct/config_tile_Sx_exp_psfex.ini delete mode 100644 example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg delete mode 100644 example/cfis/mask_default/Messier_catalog.npy delete mode 100644 example/cfis/mask_default/Messier_catalog_updated.fits delete mode 100644 example/cfis/mask_default/default.ww delete mode 100644 example/cfis/mask_default/halo_mask.reg delete mode 100644 example/cfis/mask_default/ngc_cat.fits delete mode 100644 example/cfis_image_sims/config_exp_Ma_onthefly.ini delete mode 100644 example/cfis_image_sims/config_onthefly.mask_simu delete mode 100644 example/cfis_image_sims/config_tile_Ma_onthefly.ini delete mode 100644 example/cfis_image_sims/config_tile_onthefly.mask_simu delete mode 100755 scripts/python/collate_star_cat.py delete mode 100755 scripts/python/create_star_cat.py delete mode 100644 src/shapepipe/modules/mask_ext_package/__init__.py delete mode 100644 src/shapepipe/modules/mask_ext_package/mask_ext.py delete mode 100644 src/shapepipe/modules/mask_ext_runner.py delete mode 100644 src/shapepipe/modules/mask_package/__init__.py delete mode 100644 src/shapepipe/modules/mask_package/mask.py delete mode 100644 src/shapepipe/modules/mask_runner.py delete mode 100644 src/shapepipe/utilities/file_io.py delete mode 100644 src/shapepipe/utilities/focal_plane.py delete mode 100644 src/shapepipe/utilities/vizier.py delete mode 100644 tests/module/test_collate_star_cat.py delete mode 100644 tests/module/test_mask_ext.py delete mode 100644 workflow/config/cfis/config_exp_Ma.ini delete mode 120000 workflow/config/cfis/config_onthefly.mask delete mode 120000 workflow/config/cfis/config_tile_onthefly.mask delete mode 120000 workflow/config/cfis/mask_default delete mode 100644 workflow/scripts/star_cats.py diff --git a/example/cfis/config_exp_MaExt.ini b/example/cfis/config_exp_MaExt.ini deleted file mode 100644 index ccb527dd1..000000000 --- a/example/cfis/config_exp_MaExt.ini +++ /dev/null @@ -1,52 +0,0 @@ -## ShapePipe configuration file for exposure-CCD external-mask (healsparse) -## rasterization. Reprojects the unified UNIONS healsparse mask through each -## single-exposure single-CCD WCS and sums in the instrument flag file, -## producing the pipeline_flag.fits artifact consumed downstream. - -[DEFAULT] -VERBOSE = True -RUN_NAME = run_sp_exp_MaExt -RUN_DATETIME = False - -[EXECUTION] -MODULE = mask_ext_runner -MODE = SMP - -[FILE] -LOG_NAME = log_sp_exp -RUN_LOG_NAME = log_run_sp -INPUT_DIR = $SP_RUN/output -OUTPUT_DIR = $SP_RUN/output - -[JOB] -SMP_BATCH_SIZE = 1 -TIMEOUT = 96:00:00 - -[MASK_EXT_RUNNER] - -# Parent module: single-exposure single-CCD images and instrument flags -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of the external healsparse mask file (band-agnostic; r-band for shear) -MASK_PATH = $SP_CONFIG/mask_r.hsp - -# Healsparse bit value -> output flag value mapping. -# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks -# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. -BIT_FLAG_MAP = 64:1 - -# Flag value for pixels outside the healsparse footprint (0 = unflagged) -OFF_MAP_FLAG = 0 - -# External instrument flag file: summed into the rasterized mask -USE_EXT_FLAG = True - -# HDU of the external instrument flag FITS file (optional, default 0) -HDU = 0 - -# File name prefix for the output flag files -PREFIX = pipeline diff --git a/example/cfis/config_exp_Ma_onthefly.ini b/example/cfis/config_exp_Ma_onthefly.ini deleted file mode 100644 index 71cd45b54..000000000 --- a/example/cfis/config_exp_Ma_onthefly.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -# Path to check for existing output mask files -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/example/cfis/config_onthefly.mask b/example/cfis/config_onthefly.mask deleted file mode 100644 index 7c185c602..000000000 --- a/example/cfis/config_onthefly.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_save.mask b/example/cfis/config_save.mask deleted file mode 100644 index 497dedda9..000000000 --- a/example/cfis/config_save.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_tile_MaExt.ini b/example/cfis/config_tile_MaExt.ini deleted file mode 100644 index fce00da1d..000000000 --- a/example/cfis/config_tile_MaExt.ini +++ /dev/null @@ -1,53 +0,0 @@ -## ShapePipe configuration file for tile external-mask (healsparse) rasterization -## Rasterizes the unified UNIONS healsparse mask onto each tile pixel grid, -## producing the pipeline_flag.fits artifact consumed downstream. - -[DEFAULT] -VERBOSE = True -RUN_NAME = run_sp_tile_MaExt -RUN_DATETIME = False - -[EXECUTION] -MODULE = mask_ext_runner -MODE = SMP - -[FILE] -LOG_NAME = log_sp_exp -RUN_LOG_NAME = log_run_sp -INPUT_DIR = $SP_RUN/output -OUTPUT_DIR = $SP_RUN/output - -[JOB] -SMP_BATCH_SIZE = 1 -TIMEOUT = 96:00:00 - -[MASK_EXT_RUNNER] - -# Input directory, tile image -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s): only the tile image is needed (WCS + pixel grid) -FILE_PATTERN = CFIS_image - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits - -# Path of the external healsparse mask file (band-agnostic; r-band for shear) -MASK_PATH = $SP_CONFIG/mask_r.hsp - -# Healsparse bit value -> output flag value mapping. -# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks -# (True = masked, e.g. mask_r_nside131072.hsp) use 1:. -BIT_FLAG_MAP = 64:1 - -# Flag value for pixels outside the healsparse footprint (0 = unflagged) -OFF_MAP_FLAG = 0 - -# No external instrument flag for tiles -USE_EXT_FLAG = False - -# File name prefix for the output flag files -PREFIX = pipeline diff --git a/example/cfis/config_tile_Ma_onthefly.ini b/example/cfis/config_tile_Ma_onthefly.ini deleted file mode 100644 index 8f7ef4206..000000000 --- a/example/cfis/config_tile_Ma_onthefly.ini +++ /dev/null @@ -1,82 +0,0 @@ -# ShapePipe configuration file for masking of tiles - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/config_tile_onthefly.mask b/example/cfis/config_tile_onthefly.mask deleted file mode 100644 index 69ad20769..000000000 --- a/example/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/config_tile_save.mask b/example/cfis/config_tile_save.mask deleted file mode 100644 index 82c4b66af..000000000 --- a/example/cfis/config_tile_save.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/defunct/config_MaMa_onthefly.ini b/example/cfis/defunct/config_MaMa_onthefly.ini deleted file mode 100644 index 84f117e65..000000000 --- a/example/cfis/defunct/config_MaMa_onthefly.ini +++ /dev/null @@ -1,105 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_MaMa_save.ini b/example/cfis/defunct/config_MaMa_save.ini deleted file mode 100644 index 4bd1b00ef..000000000 --- a/example/cfis/defunct/config_MaMa_save.ini +++ /dev/null @@ -1,109 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, star_cat_tiles - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight, star_cat - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner, star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini b/example/cfis/defunct/config_tile_Sx_exp_mccd.ini deleted file mode 100644 index fec79f177..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini +++ /dev/null @@ -1,274 +0,0 @@ -# ShapePipe configuration file for single-exposures, MCCD PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -## Detection on tile -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -## Detection on single exposures -[SEXTRACTOR_RUNNER_RUN_2] - -INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2 - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner_run_2 - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) if True -DETECTION_WEIGHT = False - -# Se to True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[MCCD_PREPROCESSING_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -INPUT_DIR = last:setools_runner - -# Input are individual CCDs, thus single-exposure single-HDU images -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - - -[MCCD_FIT_VAL_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -NUMBERING_SCHEME = -0000000 - - -[MERGE_STARCAT_RUNNER] - -INPUT_DIR = last:mccd_fit_val_runner - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -PSF_MODEL = mccd - -NUMBERING_SCHEME = -0000000 - - -[MCCD_PLOTS_RUNNER] - -# Now MCCD has created a focal-plane PSF model, including all CCDS per images, -# thus single-exposure files -NUMBERING_SCHEME = -0000000 - -PSF = mccd - -PLOT_MEANSHAPES = True - -# X_GRID, Y_GRID: correspond to the number of bins in each direction of each -# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. -X_GRID = 5 -Y_GRID = 10 - -PLOT_HISTOGRAMS = True - -# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape -# before drawing the plots. -REMOVE_OUTLIERS = False - diff --git a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini b/example/cfis/defunct/config_tile_Sx_exp_psfex.ini deleted file mode 100644 index ea86ea048..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini +++ /dev/null @@ -1,248 +0,0 @@ -# ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 40 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -[SEXTRACTOR_RUNNER_RUN_2] - -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -# True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[PSFEX_RUNNER] - -# Use 80% sample for PSF model -FILE_PATTERN = star_split_ratio_80 - -NUMBERING_SCHEME = -0000000-0 - -# Path to executable for the PSF model (optional) -EXEC_PATH = psfex - -# Default psfex configuration file -DOT_PSFEX_FILE = $SP_CONFIG/default.psfex - - -[PSFEX_INTERP_RUNNER] - -# Use 20% sample for PSF validation -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat - -FILE_EXT = .psf, .fits, .cat - -NUMBERING_SCHEME = -0000000-0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = VALIDATION - -# Column names of position parameters -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# If True, measure and store ellipticity of the PSF (using moments) -GET_SHAPES = True - -# Minimum number of stars per CCD for PSF model to be computed -STAR_THRESH = 22 - -# Maximum chi^2 for PSF model to be computed on CCD -CHI2_THRESH = 2 diff --git a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg deleted file mode 100644 index 4e4164aaf..000000000 --- a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg +++ /dev/null @@ -1,24 +0,0 @@ --11.5 68 --6 186.5 -7 188 -10 64.5 -31 55 -50 38.5 -56.5 11.5 -188 8 -192 -4 -59.5 -11.5 -45 -33 -13.5 -64 -5 -154 --6 -155 --11 -64.5 --40 -44.5 --51.5 -30.5 --62.5 -22.5 --68 -9.5 --177 -2 --176 3 --78 12.5 --67.5 14.5 --38.5 50 diff --git a/example/cfis/mask_default/Messier_catalog.npy b/example/cfis/mask_default/Messier_catalog.npy deleted file mode 100644 index ef07eb032b08de4fcf508418524692e27f92ad75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4209 zcmb7{dvKIj6~H$S0z_V+Av_zF=Wf!F0D%A@VJ{fCEf+Q}Ge}K(fi%99nrA zDunWq77DdgDz+6YRTR<6s3U8&j-3Igj&%@bimd}oA2U?zlbKdM_uP9w(m(ve`$y*c zzCFM5y62pGb0i#Dx@=XB?^WN{Ks?dak_puMR|f+dThsACg+H)u`P4vJg+CbRYTc6H z4~^zMo@mjpb;;T0Jrn)RYq*17Tj|%eB$n*RU?!Sg*O7@Qw1q{{wyp$y?@C2G6Z9UO zJ*|SwZ1?}atFSw_CB3P=b8}@&M!PQkRQogQ9WJxR1Qh>397tE&bDwclCSdu1(AH6 z4(J=nqclLjNNy$y{Uf>I4k$$U+;$icOckc`Be|`!t%iZDp)MQ_Fa8fL{-b;tQzLp( z1zK|XR4X~i+6ssD2bS^mr21)@q|-L|e`)L2lj@%utcLWY`f8xj0Yf~cz08Nmysu}I z@{^g-7~8uRih{w2Z!HWBN-@I6ONt@pAt6OgeuG|8G~c*H>$1v0xQBsaDf~%ZQ5pK+ zj9#xu(RJZAt&dbL!Y~#qk)qmYi%L-!8?ToXsYiGI;Jwj}Di2{ei;a+?!V|0AWsRl> zym+Os9#lTUNERCeCHYt zJUw6aL%5fLQZM+Z%ud_-6Iwr&qT_?AH)5Y0Q~eRfvY5YD%-O+3yrf8%k}`2kQ5lS5 zF$!?}vw9tm%8M`a(@7E=S+VZ$>vz-ugb6G*QHqIP%nRl)Kw>5JofNeih%kx8CQH%J ziDe7S6Z!TxZ4=cXgaC_C$oH}-GAe=*UV1+mP=gW5Sgf4nY({85j5bP5FwOf$skxcg z`ew}uH3Z>41}X@U&gz10ej=di1-q4`b^hq(-Jdoa+e?F#{-Xl{Y#R$_Fm@dV%C0vB0w`Q0Pu=9N?r10aa z1fiP6W{_UL)7z_%e`NK-A8(x7p+->k!%PNhq!_BzUb;+rXK_@tft|MwtqK197itW` zEEbzBMTsX?^IN<3T|d4!Y5&KoR4Kw77ON%ms|DMaqS5$4ebivn6}t8M9V&xgsj&!i z8K@&*oT+IkM%@MMhIoJxJ1e4yYTsCB3-YHfhMU}t3w%~Us)i{I@1A>5M z-PR@B7L@6soDj-%^Uc>N)_z{GGL!si)iAfUj; z5chgC? z3g;oLX0bJ-x6I~7j4@k7dKc!~37<-tWQGvdGVllib}C9S%nNnqSzCL_=S?yN!a4@l z6EN0g|6g7*%bn5qnc^)}C2BrGlz|ul+q+-jSP-A>&68b=j-t$X$xCiGfE6@H@}#1%&`}s!}C4|3xRHS%ncEW8jNY^fzTg z8JXpd^5&~XtPx=|i*1pj&>|L+d5Nq!9g?sS0L0-4w@4|Wsp3qZx=0U6~aLVzCyqRm{F~3&hHc>Jfya3>+iC-sHUVhk$^KK6XW4(DKwegl8D|DgmX=&0@1}8gR!+><@H9 z+WAAZ9^qLAjuT*e#Csq>?XndVKW$FmD8h3Le2oCVEeJOk2$!g6{6`KDwc_*QyqTuQPCx0DJ5Z>D8!r%S;}ikXzqWafEL$@B#sK&c5wX z9YclwMt!znzPT4B5ME^9B?1bpOTA8Mbw^hpRnUe7=8}*^_$C9V2=Fdu-YK1(=oR%* zt|gU1IL*MfJ`;oX=kB5NT zfba?f-uzXxoJl@$H4aq@J{Kw+tpw#gl=-AJ5(Csd;(*Y{{>v*ks<&9 diff --git a/example/cfis/mask_default/Messier_catalog_updated.fits b/example/cfis/mask_default/Messier_catalog_updated.fits deleted file mode 100644 index 6a9f00096170565aeb7fab3429c2256ce56f3533..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8640 zcmeHL4NO&K82bxk?Xcg61^5lKgn(d%puWZoviN(qOUV;yoAdgbQ9S23R5C zGJc%mUqCidTB!L;nwUE8!IV-Emzk_l)0xbfKNvc7R`!19<}PlRM&quncFy*U!+YNM z`JV52-?M$^Ch6l6Vxt8?D|on!!gOJNj?I!|&(0GZIf71aOcL@OCY#x0H+#s(5YrnI z^eKJtg(9EHZZ{PQW|PAtI0|zm&n)h!0aEly_^(HOFCm{TKRZLR3ptjD%+_qlmS@ee z<+(|E-9@JuqYY7g@ELn0_VZRD>u0W-raGWHpgN#BpgN#BpgN#B@ZWcUA19Oa@dm9B z$n*vyu-NFpUb~tuWLRw`d!gVkWn@W$RN!zOPJ5+1di$@QpJ)8@^Rsdu5B7Y14xWX4 zS&}W&v4Ee)%@0rF0_OL1DSR_L^MwZY!H4;MT?t}*vH{QW{rFzUwCCjK3b`h`DO++# zcF*4nulyY!*XREJQ20EvxYZhU`smoGB+vRn9^rG1jKwNtnfqonipd7O(vLv1l&P%K zd;Z)D_w!ESQ~KdWJ`L|jpvgWz+iDYpf$N*;MLuI{LbS_2yUFFz!1U?j6XRTb_*Jc3 z1LNcR9`<9t7x=h77oYd^x%!)DEt2Npr~knA%k{bVyr<7KFN(fYZ|HOJc~4)6GS3UG zHswc^f&1s;18>Zq`uoOTE~2Q~q|Mla|%g za-h=JE%N?j_uw*?bnuLd%4718+1H`mV&Nd-x*iGk9UB7p&}C4$PlbH^_CmVeV&Nd- z=89JNK<*p#3k)C#Ou1FXUhvhhA8^?Mtb-}voMsd9Z@}Xk4L!{R2OH5KblZ)dP$=zM!+^*|xtz<#td_j+GQ1-&6ZGat%$AEx3wsX?S2kT*O z(P(-PbvTh(obuFT5Z{>yK3MWT-;4Ig4*6`)85)KG1QE-xCN>W;=W>HMh*)Vk)ja6M z@syJyu$7@FVSU62dK5DdMBK3RxExb`isC~K0*QVOCn9^$ zbp(OdhJ(xyr{#o-Fcz?5&!N=z7J3m$omj$lnD%Km2LvbsqXagOsg}PA-bQ25LIe>V zY_+_lgL<|@I z07LL-iY?$qV<2@4R4q?dbnI3#Dr+G=U=KWr-9NxdBlb3dj*Wz&SaOD!L_8P^mIHNs zdT16GHv7tAi0Nvf^(agkZEeVSNc<=n#-hoUN=9c0WOjG+Cb?dqlKPu$YDEYqwBwk(z^jfXBY2yG-{#>NlWyeBp( z@5@r2p?zSM!|o1OUcz)9@V|8#^oBHrL)mTU;l)flYcOm;Vdr^9f&XuELZ^<7!rdZ> z7d$Tc4;qc|FXe?ZLlMI-okBI`F@A;7=F| BE!zM9 diff --git a/example/cfis/mask_default/default.ww b/example/cfis/mask_default/default.ww deleted file mode 100644 index c2797f904..000000000 --- a/example/cfis/mask_default/default.ww +++ /dev/null @@ -1,40 +0,0 @@ -#--------------------------------- Weights ------------------------------------ - -WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) - -WEIGHT_MIN 0. # Pixel below those thresholds will be flagged -WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged -WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels - -#---------------------------------- Flags ------------------------------------- - -FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) - -FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels -FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs -FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits - -#---------------------------------- Polygons ---------------------------------- - -POLY_NAMES "" # Filename(s) of input DS9 regions -POLY_OUTFLAGS # FLAG values for polygon masks -POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks -POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? - -#---------------------------------- Output ------------------------------------ - -OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename -OUTFLAG_NAME flag.fits # Output FLAG-map filename - -#----------------------------- Miscellaneous --------------------------------- - -GETAREA N # Compute area for flags and weights (Y/N)? -GETAREA_WEIGHT 0.0 # Weight threshold for area computation -GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area -MEMORY_BUFSIZE 256 # Buffer size in lines -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -WRITE_XML N # Write XML file (Y/N)? -XML_NAME ww.xml # Filename for XML output -XSL_URL file:///usr/local/share/weightwatcher/ww.xsl - # Filename for XSL style-sheet -NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/example/cfis/mask_default/halo_mask.reg b/example/cfis/mask_default/halo_mask.reg deleted file mode 100644 index c44f25167..000000000 --- a/example/cfis/mask_default/halo_mask.reg +++ /dev/null @@ -1,50 +0,0 @@ - 274.66813 -1.25966 - 272.54579 32.47406 - 266.21222 65.67579 - 255.76731 97.82190 - 241.37579 128.40544 - 223.26462 156.94408 - 201.71942 182.98775 - 177.07997 206.12573 - 149.73486 225.99312 - 120.11532 242.27660 - 88.68848 254.71937 - 55.94996 263.12519 - 22.41606 267.36151 - -11.38436 267.36151 - -44.91826 263.12519 - -77.65678 254.71937 --109.08362 242.27660 --138.70315 225.99312 --166.04827 206.12573 --190.68772 182.98775 --212.23292 156.94408 --230.34409 128.40544 --244.73561 97.82190 --255.18052 65.67579 --261.51409 32.47406 --263.63643 -1.25966 --261.51409 -34.99339 --255.18052 -68.19511 --244.73561 -100.34123 --230.34409 -130.92476 --212.23292 -159.46341 --190.68772 -185.50708 --166.04827 -208.64506 --138.70315 -228.51245 --109.08362 -244.79593 - -77.65678 -257.23870 - -44.91826 -265.64452 - -11.38436 -269.88084 - 22.41606 -269.88084 - 55.94996 -265.64452 - 88.68848 -257.23870 - 120.11532 -244.79593 - 149.73486 -228.51245 - 177.07997 -208.64506 - 201.71942 -185.50708 - 223.26462 -159.46341 - 241.37579 -130.92476 - 255.76731 -100.34123 - 266.21222 -68.19511 - 272.54579 -34.99339 diff --git a/example/cfis/mask_default/ngc_cat.fits b/example/cfis/mask_default/ngc_cat.fits deleted file mode 100644 index f51546da7fc88ae1ca919a3c13882fe1a7df6768..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 201600 zcmeGF2~?NW_CJoJC^(`d4mctzj+u&vn#6MsJc>gOJW`TaIFXu~17=1VYN#lVC{9`A zfF^3U*7~iL>ptto=A6A> zdpOfR16KEE`t~vd0zw1+=3oDSCjy4PGB*B|31cP(B)k&PwR?XnU}D1HvBL&W81^^i zL6Y73_w7F5Kk(}aeuF1W7(69l*x-c00SQya4gY&CZfb8apu2@1{}x{x@Ebd6%#h&| z0$z!CaUMBl_}GafUl}{m&EzltWe(_X_Kx@u{QCc;;Q#PHnc4ra>_5f+vA{nT_{ReO zSl}NE{9}QCEbxy7{=aVlnoe5X`}7VCXezt+?k{97v*};7dLm%R$gzVbObJLBJY@9n zfZ?wt=*j6{iu|Ad`Ty$tx!wQ&^JkuqxBdJ1bK4N`8$EpNO9>-rK0eGf6AwW7|MZXf zwfh@>ZQHi*(EdO0L;3&ouWxvt=-!ye{}0v+FHLx5(zt+egC`6gGdyAVguh=ebo>9s zukHU~Kj!!MT-<2;ckOQWim?73Ux)vLpAMP$k;6w1`_Ekahe!AB&i-gRZ1_to=dbbS z7W_Z_FXqSo_%HGcCVw;?JYndVkz)e_{`dIW{}=i7kLhb_`%M_EZS=px*R@a7GnyY( z)%@>&$B*#2{4w;uz>n~0e*Zmu+P@P=zCQd#EdBp?`Vv0P@4tsn#|z_&`ETIU{Qi6R zI&eI{JaQ~2RsQ$()BMDLBmVx$ZXxzixrB3$GW6Q(0ez6|7tpwsPev zE9_$Zz2elE+^{4Mk1&5CT)25il=*6ESZeMuO}OIgE6V;pk$!lZtK9R-la(jFLG{Ce zgM&?@R`xVmw#zfjG&lE{X_QTFpo;r{VpIYT4-XH^@R^e$4$l1-Ges54DBmR2@g6(K z08_?cRP44*R{!E(VYY_{OED^5k1^7gTU-U;EThuiC|UO7iAJ^HUxETctHr1ZQx=%Kcc z>e6aXiYz-~7y%<~Eu(H>f;j#D?66GD)H3QnxkmhJSO7Bx5rmtE)MtC1F!n(?CNwx6 z;tccqBy1<3L#$~8U8p0z+%$}G(2~t@Ljv7JmJ`7|UZzeZ5xO&1b%ycChkTEkj*#08u z1Gi0Q8in>3eKv%}$H!}6b9P-&&wXxi37FF|7NCiRz)1zJ){t5u)GmYsGGQu}SpIc} z+EXEvLmO30W5q_H_DtR9YJ`@t_>XjPZTfC%JWq3p#cMppwecNDq$VsG(^*_wn#fha z)HIeL=&r$0umx2Efq1;Ph{dwS(gFj)v=LVUeA7rfGL);JKIvl_OP3rL*Sg+(pRn$*MJnDbjw~neYf%DumDycd@N&)SDq-I(u$P>({Ll>$u(-1_>-aM z64G_!)`wXOI=OASZgW;xbaXUV2`1ledNEDxO4&kfMYost6MV#u?>}Z;n8*9hLb21= zmuwLVf@NeOaCbtijrhQ48d>r2VwZn8cYAr1P!uV4Rj$FVgtp*uJVfk5UkD6Zx%baz zJ96glRDy`zjm=k!9l5ZYyS-$Ej(6nyVQf>NwTx|Gy0iVWVK;Q|U>RGUIO51R_b`Et zydApiG)Hi&NH!l@A( zO!(x~Y4u{Sr`&$|!YCBI8<}p;efo8Vy&gSGyf%(1pUvGVWI1sCEHIs*7+BJ^!xPi^yWs?kwXT2H@Q-yKogI z{OOlyufM#Bbs^}Y%xBxA^d2rOj170uDT(r4dFs#ZHw}VLRfIR9@aB=544n)l%2&Lu zPLEq}OhMa+2`o_#eNdfl+l+~@M=n))giyX`E!AUMPeuywo1((zEY(W<$=y~a!a-E5 zc}{sfJH?2@phe1w3bD$hiy(&F$z`8CnqVixkC|^XXUAn9*lk1qQqlM z4y#=`dq{mEb3}!vUvc2F)&1vh6qk++A;*}glV3rKn0^Af7(b%z3;%6?Mi1~H=*+af=O{HQ zX$d`9*~>RgXeRVxl=9&FRPrFN^Tq2o!cwnyFb z@nM5Hdoxnjw%$|wjcCl2NGz(DqF%=+L+Xxzh^DANI8s%s(}yEjS9#({*hsV}*g_+% z7ZwyY3=N2+;Il-%S$Re<)CWWQEK$Gx1tapCpP9fV>h}#XI--LuK#3$$uD+EK7K`5U zJeb-{fVM-r&U43DeN1^`(a{5g_I`a#zAX)#nusMDPH82B4(to`9x^iE7cd8~e-|5o!87jd<{ax_ zY>LL4Zi$)d&xc)(XmaYbIQr))HZtqdNG^2ZzU$*|$W6YSp*F>SY{bVu}_=hRQ~-uHgymaniOq8qk1M2iPwFW{DQR z`N$^K284(Duad8l`Yo5Alcl5Da|HRXT-8^!7`Hy>5Y{h1Nxm^f%e8@GzgHeR3ihx> ztG*#(f3*SJIYO)@n&%!<2fS~SE;^E1qGQ{SUJ4?0IJ6#HEPmTjm3c7J)=h;Nj`%2% z0vt0(sr{JqXQB#+L(B9wYJcp<-1b?@<7iQ#eLrU&NfZu`f8YF|N8}0agIKx8cU^EI zKW<@o06JG`m*ioOTbIv%0nrle1I8GKR{!Y2VTq0vuZ!ZXGu`13;=f8w!Hf+kbrg0S zbUNpI48Q~o3R+fY(`cr6!Ek7sTK0J@-{5L3IjC!8p z`#;jTMKScJ+h0&$^g1RkAfyVYO6WJy>i4+D+!rz-tf^2T(|Q_RF+*uZVvRa|M46l* zcb8*eJ$EX+q09Ry@#Gr2Efx};blbOtv@*o0l-lG1?S}A1`QpK*Ta1)-3GeK!`eEFM z#dQM`!?K-P*B)b6t|VaHo;V^))opITsKnS3kr`v`Ow9Un!IXLQXm(3g7}NRT#98z> zS}W(b`8BvdWnFr9_O`#rkmLmu`V)^{_4nFE$3xsbOs`(Y>@zi#I~@9=qu)KggFD8a zX_BhSVS4 z*Qj|!VNG}wa|if@BES^=e@IhNYdo0;J1QEJyeLfa0GkYLis(PmRn($4*2IGJmEHMIesT(7)XvvquC6tF#(z=(k#z zSF8;C`kQ0PFFYF;uZ9KL*U=4sjY!TQH%vgke)IVR9->*f3H>hH-=RZ+Z*|a3#MJt+ zdwp3fBw6B7LaR9S*q^6dNW;ZM@c8z_imnD^ijm9DseNc-2sn*&WcD<*FYgm}WiMhn zY4T+Id(2=-IVOybj&>G3e?Y_`)-iE4#jDR%u=A%JWaV-hM}m*j|6UOrnIn1f;7Gdw z10P&Td`vO<)4O&7G9>t|Zu^r96LM~Ux11ftQeIo#$DUo#k2`uMOj(zob8l)h4hNPp zwMHAULoYS7J>Hmf$o^uW7rC;80O&uw)09mJv6h&c;A4MLn853(0wN?W3dsrIv6jWA zQUgjHTkOmXv8glB?e?wYyXq}0iu`piWr^vr&2yfa@hwLoGfjGDuTu%YAXa;S=6jJ! zR^G^s(1To=^ih%Xcc~>v>M{ZFO)$G=g*$9{%70VuAF6igpKhW#TH{6a}MW8N&tYNZiFeF zi)8(@0X*RBB&2gr@0Dc`QYB0+F()#`SoB9ai$yg{q?AjMWyTG3$5;xcaHTP8gC5$q zDR-h|={YA{Vl6QjQ(DhbyWMSW?zvRy^Ftc1_~J<0SxDQ?vzKrW1NAL2KV-5jlhmE6 z=%AZdr;k&m<5@OxDp_p)%|o)x+s!$WVfCS6am89T^02HNw6(+n_@zw3vxHRhNNrv0 zRK8Ww-5M6OE>?@%40J^ym|{8m>nA4-5A7&REb4bzdQ24SyHottWzz+9fgb6oIbb|u?_g)9qafvV zZ)e2)W<1`q=N6aR?TpC#gbbvarv2H|C^NqXG1LZHda$Q)VEsMj0W6kCYn<)WTn zz`!?_SiVt+I}m%r<)~#?Cfr0K(!Qd<3Sdg`&9 z<48l{x5U^KD116Kf~pqeF? zVryW~s+Qb5Y_iAFuQQxM>B_Ck@=M9a24tQH80^|tq+w#TWLdNrhi)97Dc(lzA~lH< zOu8RiwY9V3J*XF3j6L`EAWviV(@VLD0lq2LTw5<@Kx~pr{Z$i#otM`waN~IA**1pN z{U#F%D1zQy<0(CT*STZ;UH?^L%nh9`1yhv3)cpA;j__PtS3wGT^b*}$vKumPt&yd_ ziFU;ps^le#w_=;~1Rly+BBN6)lA!5vtp>F6ISVktb@OOJ|?bgn+ z;cw4!n`3P=dR|vMv73M&wD;F0-!&$O*xdDTiT4_3tNPX7VIJ&;wU|3pi~Ep)p@byFkE)v%m}`E+|(U-O)ig%lEsYu~Qj z66cgdNI@id4;E{Y<#@xDH$Y%r)=vyJwm$c^yUP0BGr}fxooB?LPfjIAZTL1vmf3#A zfMUAJhEu0y*)xG0Fw7%!$bIQmS0~)rV(X7y&w24mt~>pc`C^(Z3tfl@K1*Z{36y0| zDh`-TGU0|kvBnlmN5PPwn<6tONS57S)>Q>1uv)8%IUPW0T{5r7$g)`QLL|XQmEt3^ z%<<3N_So>V5H*qQp%P$fiH*_GvaDAgwJ7>bv2kD@S++qbW(vmOacrZgakZG+t*%n` zu4unexJzuru;!idfJ$hqZ^G!|-L9%z%BF^qqW6|xjnQzFLE66GBUG0CX@R>B+JyOf zEp&b47q-XylMczUCFx@(Y+7DHRL6V_QZ)2Uzg@`r3K4|g>f!bMW_64Sm`akP3P`#4 zv4TGJPE%U!>b&Lqv$u%`=n!BWEoJle^|GA*8SYR(L=~*FYoQ8$pbrFWmDItK*oLE+ zu<>89P9+dlLJI5wi=9h#Ds{~xN0IsU$68_y7+}Y{k|OMKw zu=L6V(o|x)JvusPBla3W&~DgKD@T?>XF$O40eZCz)fh(;;06u-0|c(m>Al=Zqnasp zbV(QOkb|WXFrg)~Zv^K|C|R1Lge7)j_F3Aaki{|~`^hyad9=cCXKtO}vM* zwk&o>?F7dM6L_H14QThg>mn<8JNI;4BKzWTr}watEVdUR-Tiz*&I6;64aD}?b!&~< z9UX0~MDs8R*q#M5q}Qaj959UIkt%zxt(Rrb-(lt0Q9ECYQcHARN~{0jwqpA`yE|=C zgC_!J&xkrXW51Z?9<=sQk_kgnfu4<9Vqe|nvNDFHO3;Oo?(3hZ3?xKSL5L>abRm_ku{(-^8h$Y9K`71mC@MkhL11DIlgz!ven~B?0%%O zF2(etoF#39S|aDbc~U*-WU;L&Ei{mV}aIOD9VRF775(bS=sjW_5UyvHmK2UH0c>b*$mQ|TD@mR!O{1U5hOGPf5d91NKZ9)}DMFty2*IXG4jQ0Xm3 zs!QOIO%vOL&!M2*0od?Vr|zGA+Pesrun*@s)gx2fCj8``y|VJ8*WGE#PgdlpJzz>m@mb<< z$YlE(PRsnTwN69MLN{Ec*R4s#g%R$J^u>tb2s zh|X(6A4~je_BLa7Nd}&!oWOQV5LTW5rJ3dh2g~v}ngCt2qfVT;ul68wMJ1R=o?R$? zOU_)eawlT;y5)=3zYAE--@S8hPBC1p))`fZGBcJ@;tNjfdp5rRtVz;{E zb8mB%0y5BPEM|6m-<;N_TK&Sv6ww!3&mfSIEm3fxjOB7C?Ec#-T+e3$g zDGI;N5Fgw4>h zLEw^KQmMpu*uZ=Wn{5zDpC!J3ElQkf*@ax7Ln?nvXUD4a6lI;rCP0ca69v9S$dt3F& zG%uuy9;A$@Am z;yZg8DT{r?4c{RfSkW55lL?I@!ABXG80G4zf+c>$QLm~9+=on02aHj5j6!Z=tnKM_ z`S$eB>bi9)Lb{{s_JPb4I80F_cFU^M|8yfQiinU^-yG{|gyEtHi;=1r#a8OsmL-1J zzTP;3yyXozicNSLBTzZaq%+}LxWW{tvB6!g?gZ)?=mU>=*Jah!KT?a*%Y>p2j>xKK z^_cBJHZ96MCaXTVhIyb0l)D@!Qjz?Fw%TH6a7wLOx8d%VeSUA0^3?~IbRTpEJ%80R zySS;NnkCMRJg%O>{tFDG-B9%Ve$f+E;58kmKOs+7^->oStGlf;7@lW|KWPR`?zF^D zJ!^?~KJs>jC75Ezd*F%ZjdTQ^Kh00i%^W4`%`w=ONu)e4I<*h0tpPwTUtDx>2lzWxK@+$m30wRI+og=kY; zxEv>|b$FejGq5wiT~M=eP6?jZJ9-qlbZ5jzLCO42}N;0f4 zqa`k$xGc*e5dlYQ92a52GF7gU6zzr!J7SDo^_H=-J28%98;xCFd9H9U#aZk$4>_z4 z95U0hgCm_G2lf##Duohaw==}%0gL<+E-qoaB^dJ{C`J*`U0i}2YT_6Ue$cvH_{iH{ zaN`Zm)4@nM+BC>h6=SM!1KI+dB`(!Kz7T8VT$m6JH^rse39`(;V%f+n_7XBiRo*|& zJwK{h;%xRbXDH4p_-j|frgow7MHE}QmqLl#lT|V{Fo9Czc*G6=e#A;#LdwMV#UAVz z00WMVLYAFAnh72hK^L$3*;_F1;kN|NdQ|L{RSDv=My7kAc?J7oWfycZP@g_iT)~cn z|L|}M7#;jqkm>OTflLUrmMCsJP25IK`XThI6NANaWF=9Bh^Dxj8822;i8at(j1~+iaEnlm{5G;vN+IGxWdvB*RW6c`?$xbBhW~% zZAcaOa5R#{0y==boNOP&egme~TK!*JCW|8Mo5E`pc;JCn*#nygwrh;K^lTgFrRK+} zMbgk_iR&Gs!~>sB>};R~0Sn&qyPl<;A|hJiMs&3J6Ip!lAkn6{4y3PReH)At;A4qj z-`T6G;-qWPF%M>X3jyQNM+Yt} z2BstG95{w!!1#d-4W`4g50*vOmjpsQ;kf-~kn}IVh35`nYKg1ueVo|`mbn}9ZTKZA zY!3%NOS$vdVQ~wg4NY~+zJu=bJkmcvNx;CgLa3^bBWFOWW9+tHqN@KxDIPs~k#ctn zR*9d_-{BqxOigk7OrZMc&9SVUKOwz?ZJ&zHLMAZMdpMTVA3MCzgKT1odoU0P)DeW{ zl7a4ecqrfOX|7gjiTnFn$#y+Lc>rdyH=x{{AN4|$n99-_W7HcsQv+Stx3cu6I%;aE zMlPh`(xXYL_@#0UCO}HKcz`4CDQlLwIwebbV0BWb&X1g7g~lMH1&fP6en<>ied#eO zR19D7>ci$CJz`dgCop|N3DUbBeK;_oce`GZNgI6 z@DIcg{jfx-Jrz_S5{TqEmVzqA@4mhqQb}YFPhI^|B}T?x(va&hZhLrcD^`9bOHn9@Dl-;3Rj_P25BUDX)T_!7 z@dH-#K_GySDa&F*Hv}sjvMHLxQ%4_oQgU zu5y(pMsZZ7oH0h6MXwED+Q^SoDGzJCi(~UJpGN9^cU8`ludQb2Nu+K0pgb{RP%nZ` zU$`v)cxN#J-35tdVBXj~SRh8bk(Nt2t~y}Cnu`)_k8gU|l3(&(zpVVWr`OpR8Dd1opm9R?MdtobK z(ArkKea>m}wmz_&N>VDX-fwS5!W|4L@GR;3>Kgf4wTmJE*$sTAtc>kiuQP$XjASEM zdA5yg3TK~pHRQ@?7OG4%4-m)=-DQ_DJAuWrwpF}^+OYkK zyG2z&BJ+Km$UcNAek;<}H?B}N!(ugFwW5^rrR_K@Q(1}=C<;oBHnIA?*ljAntt0Tz0rOZu zp?J{iDfS#B0T|Borrn;*+vrdbOuOcZ0_cJW(pCHpjglegKOjN7^05WQYHCS04l{y> zPq8|TbF8SZy=NY>P8HcdW zL)#?A@otciZ0E;E7ZSxZapG3IF%4*MaRsype6xXp{?bpj1&X7Eg*%sWWEmpl*ih3MVrxF5~V)8u#DQJMwe(}Rp zvKewpFfy+zWsPp5*8oQ(IE>Es;6KWxCT=`xR2MoYjvgY;><%fL_m{5mh!}3GzQ#iAE1JJod zt<9n0$Hl&^3tPR$TiN!kbKg+F(AQnotQF-{MPkMaz&t3%YQ0=bEo-1JqA;+UpWjmN zB1uik0hlRk;cin3Qa7-L_FV1S-m?7Vn(Qc>Tv;3ElRe?v0_|H%)^2-Qdj9+s$0;+d z-7!b}k~opqbKrv#(T8MdjJKn~lyTG^vr&4UxXe5N-ID%sg~A`_9a2$(dH6p_6B|!F z&p2RHl<=9MS_OSzpgs}7^z#(yIXIH7p2WZ&Asp*0Jzv<%eIYZgxuJp6t`=uD39TjT z^e&c-cDH35?3Y>(g47cj=fF@mu{v12HhR4;hm=1dt$$^stnkJU9NJu^_MXY&<380G z80%8|-85AR-39SLi74s6c$!nUbuph*VH~w~20Qgy58}4Zj;i~0uq@lB8f%+M%GF6o zRMl}WfL3d)t^X$x>aChO13WpHkfosHYr()K?-mzO;l}=yC57awjQ?8$|L0pJ!qBFSJuNWb6MQ= zq7opQvVPz7Mm6MucLIm5@(7O&sj3a;Qz)*(p~2I8#jj6Ru^q8LSHcw9 zww;!Y+g06nTg|OdB zvb1m(m>%#neDqOKjkJDloKYFGVB!JBC^0w7{ssX9hPp2dh|iOCk+-~z{CWVzN5Go# zvTo=;_ARP)mkrv*h{qSb$%BPWJnHtp;0(msFNzOl+MwTM;{ zErUF|Gj#B=WFss_f~v&2JA(iuM7F`V1zI^6X!u>_g_|V^I+~J$^X0hf#IXX;sfK| z&Tgn8pFV@>`3=_;F7V44;_Ql7dA`n$YE%(NG;dFKx9mo5rpfAqmQj4s2P5PBSg9Sq z@F)hmnz9j&oRvB~j-{~rfh|Jq9XKbAfYC7)ICPY%j&TkWwD$vRC#dmQi@|GT6Y$8p zAWI?E!H^6MLOw3D;ADln$~M3K{bJ7W5i6T;1U^VpWpLq=q_8U+4LWVSgYA}TAl#s^ zY}7DPRT^@ib@5>6$*E7T!Cf!Z*T5QOT@Tydc%<<*8W82_WFu^1%)sRh6xXex5$1){ zam|E~vT}_O4x@2{l1c!aC4)*YlmSQ)5>k``p;DX-K*|Iqw6hxr2CAB9>Oe^)pxlAe z>H@|}+Q02eQP`wqEjt|@>_Z++YF1EFP(0Tqwh6YcOBY2l4_FLUc1NqyS8^$6=?j-l zwr*6Du+fexI_R1_z!7gu6j7Xv1Yj>G8}XPsoCo7f65A9z&5d!H#fC0PWFV9{_+ZIl z>MqEPC{cC2n66LUY4w9zWU28;bU)N3XtD3Xsd&!-RrDmU$%{cMH6nsbu*I58J*`ro ztKhEEEFxQ$-{!}jW3f$NU2|{=QsDrlaWu^+G?rqKNCYSczkC(ryn*t)qubn_I!66gpVwL6)~b_+}IHfbEq>z!7!0gR-8F>s8I6a6uP2Pj>o#flse zjlkUr0^nP+B~rgp7;gzW+q7k8Z()^c6bexEh0B&96@-QUl*FQnDVyI*P!iMgO0WlO z+bl9gE&Q2p`Rd?rj{CPS&3d?JVagVnc{!&b1=i9fT4ATT8BXjH(o8bpBZQV{mB+Bo zPC}frE-kaJtK|0jun+j4N}nS+8!&OkZ?$r*PI-#{xDQmKizVA;`x-Wm69c+dt}X7G z+Iyqigw_d(GN|7=GBVAkP1$-xiVVUD_Y!TxW$VShGN_9_^9_5L5>j&31w5dRk)VTs zL%x+ThXE)ZFpqx_EBa#ho=PxMT7dlkOvlQMiNSbFd4J zXc92!1H%8cMC%1J)Uq0Fc-t6NV8X{@?3*w$A!V^`ho>HV2k%Vy>##Is+p?)@9QI8~ z3W%tJL2KNTYsk?$VA?K>weRScD0GQ-Et8Gskm`i$I)d6^Gqwo^VCbQxw8IgaCb;?d zkPy5xLmWBx4X5@BXmZuAZgVvr#~LA0M{?UX>s7qIO{%4||MaeGh1(0zLkC@lj!`lg zcZqFC;lS1d5Y+QxdZBJq#;WaWv6+Yc_9%* z-Vgb`k9-{a`!`^9%>%7r%#i!;0l3r5eAyD~>|WpjGg`7^%5m8oxg4qzOnp#CoQG*X z@eOuEE+Kt#eS})xI+NG?6RAXpf>ml9kivvyBvW?4?b>lTg9fI$MTJahrD}h)f_$q@ z7@;ENd+8Nf>;n<5^Bnw z^4ureOc-0NUdPq|iDl(NaJ}?(lz50VBrQX|j%PNYsgALb6Vr0$;~MmNS8E8Fm#mD8 zF@LFo8>s@j<~v~^e3tCoF-JB<+Jy)JX37wx)2HBwW;GBBsHY3LkfOrA|CoCR5KY;! z;cj&tcOwBzcNd*1gxdeY1)&mPmh6P<$MNn%i>B<Bq#ZYhDiiad5(3o! zcO!RhK2%lK52R?nbiu&`|JlPC7`w7_+smpJ&eYrhQec5DmEzQt&+2mjkxIIBR*~vh z$x#hm;)k@Hp1o&q&H&Xc8H%uMv}CH=4WZqS%i1e)*n~UTKv#=-{gUZ&!&A5`>W_dq z4Mx{b5B1gZg8tsJK=4x_`-IzGC*5^K!F4NUx2hjy3A z=L|3G&?LFqeiyuFFbCZQ8Hly*a{aI!iYXivBW+FDrT+!{Ev!9If~pObVVzdVCU{7r zga=9-Id05hj35VfFQE19@?FoFe&Y?`>*5mer7;Tct6-GnUqQY#;^}ni+4q zc4!#KBPGHnR}ii6oCERTDo_77MzzF&c0#(6;`Hfr4eUl3)$xNa5%zRX<%1^>Vn9GC zs2>%hd@xu%EYbO`2q$@dXNn+gJMn-`ENLQeo8WRW>T4sL z3CK*ms}I&Q^l&^0F%5GV6xSufH>IfYxbHz#{3$-n`=@0PF0w%}id9Q?ZImyYVrR94 zV9JQn_w72pXRy^lLH$KKIn=u#t~V& zNCv&79|XyzDv>yX*JQHI-e*Te%0SsiaVGllR$C-8W~+CfFDPLlO<;aLx5a9QL6SU`%3tR9~D# zpb{X$>gd;3z$euZFbpgbhe^E=$!;FaZrPs?E^s%oZdjbH?lH?ajKKo=K1)U}ub`HF zvS0|Bm)1AIK z`gj8~?TQzd-nzd`P?hs!kI0%8oaC2=leRj>dVCOTC&Pp`8W7N0vS*$BYBOGt$ptV? zcyG5;?XPD9dTvT_s;zyQ+^Bi|S%C1No)%*1II0>;|*u8^ftKh{Sy zA=;9?angP>W^BX*%9*k^)?%B{<1W%}Fl9fyQ}H{_^TRLHBBErJw~$TyC$gj19#K(; z(OuNyNQM;fz_mRe94R7QP|cKm=~1F5*BBpaf+hQ2sG}r?lbz5dlft2Io&8R!#v_?& zDz&J-SeoCNH<+&xbfwO*&z&5{Z)G`d9w=eSn7l-}xbb|R(IW!D@0Yk!^kGE?=0Tl-l^Um3VRPxB*#4C}t4vG-W8i(5z>@vnNLLf* zq;T)R@ew^N!hRR2PC(ZkY}D3`PWLY9JjabEi$zD?QgnR~#dRb{zdzJo`)zBsIx~$v zdR?u)(wSEl;A6>WdnVhTqanjuWJXg)<9rZ3J%Asyfu8+3Sk=REQ%KMTdiMAIsvdH% z&_l<1^xNr9Rh+q`7Im0$AZ2oQ-gOqsz@Ed47@tj^YyblVFT~_W%4 z>~{7GBYiG(yjttdDVg#S9s~Z|tEOS5N%VqpH^}M(GUL_8CoA&Uz$FG$4RI@%df9JGzqW5rYH%IA6$Zj&*syXt!EY@&pt7Ysu%aeo4clss%171M8-% zB{-x9kva;W$J-ryLu`8efqDptMZ2x+bR)5PG#&W)EqfV`6%Y@I1=CZf)mNAXo`){r z143dSr*iIF#`kd4apEy>#(lMHdLxQc@(l<$ny?#FIH1%4GjRHDwKU`jZiFCOav-MG zJ4-AE(O~+0y5sX=DPAKR#55MS!Cphc%ytFtz#)O^UfM+pN$t7Vb}?${`jJK&;uLvY zQx0ksmy?ML4@s!Po*UHKFDDc2mTU^K&^9N{X%nvx*zKhFh{Xi{lnI z(!PWTO56xmTd{`&9J)`A#kTRTnWMPZE+Ab7RlKft+lAZ+S2B*@eC+ha>$o!j!ICeS z4eaH30F;;lhb3dNPgnydW#bVr_)Pi2)bZ-MW7jx7H1y)oT!{*XT(9A54?sd_@d;C@AP!4jLM?!*WeN_mb>Nn*3<{NY^pN& z0tl~kWaupRkt&1HeeT6>6Y7&qEE$)Vs6N00T(AYTIa3bCl6Xl&ebFG7#+Si+CfhS` zIIh4IL64M(D?Xy?Vb==`sUd?0w$ZC%XiS0>5FjNv)oF*TNH<_H@PTrbj@fVNJgzP= zc=di&tz=3IEa0&!K~>u~iH0Nk!sU?qJ?$BAC4MN8g6#RSs&8^q#b%4)c+|Ag-M9fk z5_tT3iAQieqRWum6;v)>)gWzCN&O+6y`9JK#@$Xphd!1ZHZeoZ#72nWGGYACeB%df zq?V|UDl*V{rDSVyC29N8h&t+3%uonAl(XbWY%G_$vzn!_fnNHtP*uiuEkx>OI-*0g zeH4p#)Te5u9EamkDNVQWJctP+njcqHfg#g1+j>;Wk>jd7a^u0eAzO0jD@U9;$X3lakkSAi7HZ`p^rklEsy0ae$Z|h)gVZf!*&Y@ zx~Y#!?4#C|v|ETa<){sJ)q2n0*)QN@$q`8YgtV`>nn)NVyxsIhdE=aeUV zp9vH2%Go{~6}17KtBh}vMOUgM+N>PMQgF++U*8ZmGT@tX^a4NUEi9Tz7p^h^<4biU zBJhK@>4a59syed84@->0bCywPQ7d6}=8=F{uk=`Z?oX2_EGOZp*HM!nMAasoc&V0Z zhhyRoc}$#}(9?{3VFxJPHd^zb^)3U+3%=|%mz9{9#xNO5L&>c<3# zw&cX~@y_GJ!-Ws}pq=EG1oS}z@WU2FP$n^*_)WAkVB)9luJ5$~4^htVJWZG->XvQ{)@wbefwIUBWe?7pRFJ^ zmwfwbz!)bAPuh`zdQrRmzl)vJKGB(py)_iq7JDr|&k0L!!(%1fY01er0g$LC-#WBs z?i(u8)0L5mK|Fv;Bw{}1`I2z^<&CN#j_0(}q7(Yl7_!)uQ{$aD#Az_b+iGO7*FJ6F zBw~xAz=br?qtF?A)sK~9u~V@EzTd*1*FZrOFw>Axw6N*tuiKm@r@!SRPU32p2Rw%_ zTux6uBu=i~#S{3IbcXfKR>#y&$n*lKHu9S|Q1+Ai^0_6GlE#EZ<2?g-jd+0Yj-Rm- z^V5ffH};%!;;;Y010Ud9a``iz<(3J5Ng++cv^XwwABWTpDv`7*R4w>kzaRs&mV6V< z^abpR)3z~I^QN4BBGRbc{#jnlqsre9X5b~?ws@!JJb*zLFs-r1d9i*|ntG8LLHHro ziQH7j-F?nzIL&F_;Ct>rl3157dpe1MyWGbb=M0a=ji7F#F6?1biJ3D3)l_V@k%15a zrkrsuRlR{HL&(SseddJwDiLWHh$Oq3a_02QDiIqYAZUxt#5yVwIoNn01rJNkDCO%6 z%-JJ6;q2oiWyX?)jwo}IZ3?@Z@~xKFbGjfa35)9{_Ev|xIbpa?S%Slovq$8}={wr< z=`fg}yPWyheW!IvK1kzugfOs{6Zq0x?h8@Ek|}d;$tANs;K72$F6s!vwSnB@f&gIm zJ)NQ07=pE^4>4tO%l)e1vdUx}-H?-;p0i)VqM3j}-UtE z>yHx#I#?(eC1)X#xdA*NK^y3;7mqjtOYUeg(^;*Wt5gukruZy5tNJ08S~9-C4G?=c z-f4y_B2Y}1nB8yM!MF7jD!N0NjkEmC@!JA?I|=FRL7l@|lniUkV|H9&*dWhQ9H-D2 zNS7Tle#5y70;a{PY3dt1?MqVFm9tLfsqezw53|ZSl|93JaqB1-_5f4x7<1a*H+wk8 zDcf{bYdOvFxcyWj)|9il2?r^4?i^X{?5_^nv^t>@ zn($2ndo8Z~2f!~-5Xyaf%YF~XaDYK~O37Fh#g;r}n@Xel>|YY>bqFccr#M5EKaSa1 z2wbSHwVmT#!F~@-9lzC)JjZK|y#RwJO6ahBMB6!W9`-yOJAs4}9;tD5O41U8c(6U@ ztiEn1m%P&pK9-!b`>s6)&z<52t;-yoLn)8*4w+Dn?J>8Ek9c>reo_}`O*toKsM8w# z$9Y$Y*?AGp^CO1wDI_m4;and-yZel9Io44Hgg&u&C}0c^7A$ryrcUo7Z;4PKv8J5c z+t2P*c_X`lot^SYwi*Ck@LR3^-1*~eYr-$w1w$V&eT2vHLi=q@SaVUu<`Ty|?eq(E zsEKKd&CMuO%duL6gc7Fj^i<2k_R#D#6xA#_FL;#N5d0}OF;;)xAIDV9l6f+lFtyq! z;}U|OEXE~L{Vo{4eA}8N=sqa5YGF<#%;`wEOtMF6^dVIqz0^tsuCB7bx&0(swuL^Z z6^TtfwlU{t?3|s4O;O5{^XnI@b+7m3oIy2eb18_^nJGg>RdfbWu8*8QsXMxA7D{Ta3jnXcwr;Asi~fKmcaS1?>*2atmYKL(+nfbE@2; zH(8fnC@TJY;` zdp%ylq7q5umxWc*>@xbkSV@V}F;0`h16;y`OBf#ZN?b&tDq5GC+ngqNPa5?#j)jZH z*q-Qd$;hnSqQ(*SRvd026LOVB=3V=%zaE@k)VtXJ0tsfSqH$D8ciLUmZ!>_0Dd#77 zI5RT}36u_)MTJFn8*~P+mTpmtKCkoibS#=tT(_tN-}P}u;{lA5uG!YYe}}&KXtLcD zlRSXZ+Ag{kuO96Gfql!`F6ws92}7GB&oN=~FGcn~Jgi7!*|$qtc-vjk_DL*0Q!Z%| zZHsx=Ne_*5iHxzkVqdHhq!dsev?TJD-393%;(;obT#}HjCSflLztx@6;;o*Jx6-fj zXdLaZC_+g31IG;tx<@RQ*1T2Ni4r$Jv?**wxp{L>U z`BH2x;sTPtQvCRRLSAu_Xza_bOB0j zk7e5nRXgN^NEa=(V5ofuyIut#KoX>U+()&=-;p7v!9+-J7->I?GXm}shceVuyvbC} zB_;i*V6|8-WLyMk*hA}*j=MQ%a|9&QHkX@g2e$mu|v16z50 zp_6c^A&JyRUU|EMlYlGk{;s6`${&s>4@``5(M*}g%G1U6gOa%=N&(oDZPbIcy9_WS zKHcSty(vytQ|r=;EVi;R%c+5}60=vG#$H~rk25duK2M#v#H#1B)GtUM!^paKST%0F z`UOvfu5`7i)n@{YOn4vKY8)V_30Tfj3GnDHSK^QTj75SOKWN{sE4PUF+NT@JA3jmTJUeS-JOm_E8}?3h1wuGvzx=Bb@1%HuLO)OROE1 zsNTdfAs#qjMoX?Q<72;%#9c1LLP1Ne#g|BPS7$%$F*tI3&7fAwHPtyGO4(t0%p~JXmbzq*OJe1KqZXI55SB63$_J)52Jh0m3*Y;6TVNbZmqpm5^=> z&sVow=@+Qr7fWvZ5Pw*rWI=;crp$bAnj=5E?Y8MAY{s6%on{XJ!v`KyH#$9WM*^PL zZGYpOI!@11y;uq}eXmE6^X{&8B$7&)a`TC4cGdJYA^^bffl#n2=L(()y#@k4F#R;z zncebB!+;cgrrcCD#F>p%<3mDLMu@uh?)Uh)WOlJ>`8H=>YMfXE9$Y006V`H`zteJu zz7}$8Q}I*|j+-XwDp7K4M8wlxc)SL`CB`7cd4#URUjQP4O@#NK&vNDlXE3k~!m$<8 zrt~$Q^!G5xQBiUWvgaTCsnhywv8}@!s9E6+U9&AqzMp%{nb-Hthi`_;&6m@iq*Z;n z5pq~=35;=Gz>DOF9&N&{SH`PUtO{=EIEC1mqnz>{g}hdW2`#x5f7+;TEWgP_e7ei6 zb8b1kpX_gVK)HAdx(|jNR$EePa~=l3EV&)edSAr}L~<0wn(~9qp-zvI85=w5gRSG8 z9>~B$f^I-N_6-$bcvJ+%$!ezDQJkjmJSQ_{DIZ3r$oLrxT{AXQ?i`k_zUefHoeeDA z<*voP>cHRv_v~WxYjvFUU34}OAz{e_{dUW@KWxJWCte5{ zJ$*DYg^?_|8*9Qk7;jMm|AQN{&)j$BH3(v)3~U$jg*|SqaaY-Mag|tx=dm7E*@Nbh zi}@Jp(-$T8Cao76kgAB&1XJ$)Dn_is)6D^{u-rX7+IbVtNS;1@3O5k}-~SS7)Y6nK{W5Ucqm7{Gx+cjF)7 zMp^63I}9EEwd6+*au>MSM$l2plKafN&g)He?*OKz{P;nvI@zEHdoGhgYJY1S3BV5C z6ulz>N8y}OAIy_;!GnPvFvh5*80QdhdQx+sS97%#(;nM};~+e@9dleN!Br08l2HL( zi%v%e%eNN}Vm~{+79u!D{Aop`)5veGq!wR^>>4|l)Lt_&hz_V3ARJD#`jf`L%V zk_X?r>?B>PX_RI%h|OCv1ii6-$0Twx?4H<6Hs4A(QM;&?H)J!h;X3;1}j`WL%;8_)C3O5|%gRk>}2- z@3zil-{z9G$9yl#i9-S@<5a@br&x}|GjQOcL+WUFupA#dftj+Ak8Wrny)Z+;59CTf z&uk;ThVwI$IxK&{yTw^E7P2lks1N#LM3g#JcLF^xO!`>zC?+GDaCr+q&=)1YxE7)o zZ_8vH5N*k0WB1Ck1w5hAemOQFP?kk%4Tksh-lypSvs_{6}&4 z=^HF0k{~6H#eUaEokYM8p%+V8QAc{!`H_lKd_Wi6;r#}04|*Y_{3)bzahPLv2OV@E z0BQROXBtk+K{0LQGcEhbte0zXBlI9w9&ge>cEBY88=k{w%G^F_PSU*1%v2LbA9Iqj zlG$?*YsnLRJ!E?ve>v}p*aqJSi0Pbo4}i1^dnqrC3{ojQj{8uVAR`5%rhEOy2#CvL`NEBUp4Z{l;Ux z>`{UL@~Cb=Un9G5ZdqjxOV;INO(A>WXf@db@GW_=dn?&vRvveKY{I-Mv8n@@Cb=T{ z>t{TjW~R<-GxU>-3Z;BiKLiGSOnGvDoQ!;1|J5DD83>=(k&$>L5+aEYq&(hPR>EEZ zF%6~|%f}x+i^NH{c$g4`aQVaRZ8Bq^7hk%XLaw|T5F+39`hcxoK!p5yp0e%N7uhe+ zxVt>rvX4l|g@;zU3q^_aJQ=w|-%ZdsR<083acU2+wO>wVt`QsXzya#(M)+;0r|g28 zI)oHeELm7CMNS|66#EvwvE;WakBbPr2253e6eaS7_^99v1;6f-zcV~kf1K6_5AvQV ze+)Y!=MLHBT1a%0r=Ca?+c#%X&=DU?{)or-=8nx2rF0YPE(`mPai-ulSR8P`rk4CJ z_L!51ds=Xk2KIeetn(W3V}z7>oSqph4n~}0S53A5o|YOMRp_&Rjtc z?S>zErHj2dVMZnJS@QI2w^VIp1PVZ4ZGYHvPI`xpWxv2`mOS-ag!;bo3bMMvOuxIA z;JgtrpY6do9x3r7?l*Wp&u^z|i~Wesz#I3Yk^r5>{x~;OdXJw*47Dyr6%r(#qvKwR z#eV<(Drc_8heiNOtR$r0FDi8AKIO#?D4zWC{jEGFCHvuyW_MW>xKJGWc`gOMZc#

4h9YH;BjCK1aktyi9r? zzQt$C^H#pBd$rgdmOtYKs(xsJeBM`jmHUSF9~8T`SZDG*>9{Lg_Q0t9wl6z zA^Qy*Nt%Y{5|0aS$I1RBmwVU^KaDx<%*C-2s7@mgO4Lmk_e(aYSjr<+F8vlEmw10g z7zjFaX?NI}SLFydpbU1yzY{Kw_my>#=LZHdk|i%rik5Ye%qY--8zq0SGUW4kK?cUr zv3@ZtS>)p~Z6*u^aH#T0w)hrPIMk;x%959QB#0Tm`A{fm-(IPaCkOrBf-MH3DX$Ek zCSSl_BarGUSMbc&ko<=)E}8P`xb<>;$vqtQ%hd=UIpi;YFi2iqNcQ2=1j@o@p~ zurBy2hIQFVt}HH-EeF4H#ErDrD^Cu=+0i5zD40rI!Of)wd!r2C(7F^4xFv_+nky0T zp~M$ia>(_n?l>*RP=J)YJJzq>A1^H&(|U;GYTOvv0PjoygKiC1)7OZDnDdjxSnSm` zodxD~T!N((KXqN!#NrGlq$=p7Upi+LW3Mq6eqmj%-boQhO8zPWN~y68J$I`P#3(vInkocS5YUBK3>W^AE=#=OrEh1~*(e?ey#1m&d3~ z*0%QyQSw_|qPRHL*a;l4Cu!SPUiaxKpKbS-I}+Fa6(kZ%GHBrlwinEpV7zKtzf5*`fw+PX=I&f@ts<0}Jp-Br^LKt+FJEwcfC2POsVGV{hcA28zYX4)Au~#C{)4CwvgLTaF@eUZsl*2A(%P( z>y~ZFo4EKn0+)L6TW!-@7;wJAB@!|+9unAg@E*jW*k^#NV)h_-}eLE%2N2CsS z5439&Uf|KPEqOYMP3L&wDIvCUVi!=Zl&8Z)FQlrjm*bAM*UiWfNI?T3*;i!L<6?QWccHeFt* zIxKcrBek=&)h|p6cFI_$Oahp$^5?cjqJ$j@B&dd%F8|z-D7LTyUnlmponk$UD!|rl zUwll?zgM@0X{HK?y*=Q7_tn$_-PGZeWpA4m?0uCHiU(jY9V!3J-Qcw3>Tih8QGez5 z2`_mjz+mk0*MiaFAhRS0M*Y7gtQD`(Nh^zi1z@|1oN=U&4#FPVgnx@_v8MFVy7>5; z@hNYfXsfSR_W0|Gqn;x7))DHb%fFrr_H1<9i$W^@dgG*L1D&*zg2l4r-w&tlYoFCh zOVP?T7@r`YY0*wQN>`~}H9~w!_9&1lXQj(phE6No0X*7f$=c!t;?sbsb-iV6`DC$` z^wCH#5mdWppIE^#P#H)w{kO?Dv6W376w?ITkB2p1yML`VGLo`vKavn<9@FDO6WV`s z2z(ivk7d{+ALzEq)OVb7tZ|)f`+XZIJ~YRG zU0`!(IhhA0>meLc(gBv~yuky8hp-1W&9WQ5S}Irlx21(Wm4Ya@d9L@Rf9MXy_WNO>XVIr27sJ8!yAYf@DE?<(N1tUkxTRE_Wcf!qTk~jeQ@r?~;07%P zNtC#C0^3Hbl^)v2c7sQY#0RlD|Cmcyzgv95OdQy{$F&=*ND&(ls|be2x?-_`c_ju? zM3k!x_C~Y#SHuU0U%K5eEJExZd!t6GaWo=V)+!EhdeD{#NVgj|J}uV_D80C-VmFRR zkOgOkXwQ|=8tleBi)F#_)k+rwtTE}N3a&TfFNAM}Av&W6u-U$oW=-|iD>2{;qWV!028XcAzX~TXKJd0V}@dGK<;BiZ|wUK+# z{%!-Z8$TuF>gwNgl@eWITZPy_2AK>TMK#-Q_|hCPW#t6DWuuukYFaDPNZ*^SN@%j+NNdht7FHyU$PbYML>53#ulX`=_{iIMa{U{LTOaO8a!ckjWVoP|9` z9Q6kD4YhJ`C0HPky|Qp-vgHR4kBXqi%(T|^oT4L$YX6JCv}s-TSY1LRZG3~DY;)wK zR$t?2e7z9s$PWWo)kIJ(d$(BUUfqEyglj^bDXdC@!8F1qO~%RV?ro-dXrzr#UXa7N zm_u>(1II&?#X9Z*U#!x2QKc9(y+%YKh2Lnq@pob3+)qoi)wTLfZrLiYwpQy;W0q=_ zP2yV0CM3@bbw{vv6HYOv6-O!anby$cWU`$8>v(-90H#!VcaKb4Rjs_HklJ<=&vFQx z)gP)efbjk-nYQtm`Xi-Gd#*HI6WkK;_`ge(#mck?X8D*>=&S zlg78Q{GyRxKIkc8PKf!`j}rTYCzZ=^)HhvDld8j>QFM+T0>1Zs9c*iAZ_B zLS9b>{~!q39{vL=Wv55B`vN!1_HPmC>Bc}nF%4I~^=~@Ub3Iv#;;2IKsuEi~quCeB zfM8106mcD))Iy}eI9~`s{j%(U*2S_V zXV74%oHY`{IoYzZTy(n&%nrX#v|c(9Sr|O!qxLM8mX|utsri zJOiZsAc$(&Do7_pG*tpf(nMAaUnf1rZaQqOyp!{Vh&6txXVZY&i$s6=Acey)y=jqk z2Yb={VX6c?aa1hj)J-|S431^{L}R`U7fS@*xK%#O+7_ytF2}zfbf-`(W#L~b=cg5_5s8V-}@`TD|u&kDtA9;7S74M@qdn|?Q4&ib)NkApl) zO0x%M$$S6)R$C0Tl;9TDhfSB)8K`OOZc*_123-R9l!zT63Q9UFhAz|CYC#U7OQbwF zPqrXC5MoUfHqBZh3RZktS0Zp`g_&Cqc_Z6eW zEi1)pmO2q@da0%}S*UtLw@SsB5+CjpAMwprB`_g84g`yjJV$l^0dxqX&N}Nj74DXx z%0KZk=F4YwX8;jkOJsigVVH9l6HWVE zn9ty7c3*%T`EpKOx7GBuG-ux5Yt=x&0vJ--tQ1=xouvDZJVj_)b;io(Xp;)5e2rtk zO7TmzdHKs!$zRJ2?7HLocC%9_72kUs@@igg{uD z;BWTiEUS_M8hZumR8^K8*lM0k_-wQ4gVZO9Dlf0Ks_3;Ln7|F}5G{jP9o+>XPeBjt z94~|1vyy;=M?WE(bNp6Gp&C5C8RyKTb3{`UQh~&J6k8NC+|jrl$O6Eb-#*X*1ALg) zBTNosPkpuWtqI(~(4CGmX@pKD7No4s%W|^o%k6#YkF*1)R){Ov^HexAu;yPB$raNq z9k^AhmuhA&@jmj6PE>2G9>9az%jUFHlQWlal4AwG=%+XgA#mSz@$->qw3HIXBhXtZ zHmuQk6x|Zu0Fh5Vlnm2-X4%dA@0QI7?e%yLi$Q2`NaVlKPZJLSI zEB+vl5uP_DY~DUbJVtgP^ibX(V>iEHrtE1(C&rb{Ck4wMydhKLi}mKu&yty~LbRhW zLAu?%UyRtu`47d^;E0PAPqFBx2r#BZ-#y}~O|NN!sWK>8JjWWna<)bq)ONqj8@N%g z85ro+K2gNZF4RvEuuq-@^y~QZ(kxd^7s)_+!A-q6dPH_#wQAXmK~Il#zLodQ~^WmiU9G@lh11PVGPoGRR+{LJhP;$ zP>pby8|i&?Qa-1-ogky=Ps;(8q2puA1aDZg-!HCi*g=7FoQGNXk3w6IVj47T zr{`JL=oJp#DFsDkI*ZSlCk;`b5}(eKQ7=r<-v%^>fajtPn)EoD(4tY8Oyhl117r76u+5om~%NZu;GN_bx@XP8!oEL_w(1e)*=66T_V%BpBZY)#Gttgq)gpR9DeBTJX&tT>XOn&S@SkX73e= ztf3&mL}81&YsFx)N1?3|+v1u%p0C{drq~4Ncc)k_-OmZo$3R$VH6cHLhA-AzPB|^- zpR2O?uj+Er?H2cCJE4 zZHtWubUHdD7!$VGo8y$SJFBYTsC2s(+14HzW?stEgqEur2WLK@b1beKf#+r z#)Av^7UCGY7HWec6JlWkOC30f!Si~KcmD*1_Hp>Vx1Zy2p1@( z=;FUY1h`A`2FIyDXGw*5LxxgWb}*YM8*er515pL0vkFBtN1MQ)zPS*X@U~}-9t=h~ zMd-q?7=|X+TRgtVyPmU7${t!ut9QnUZJctALb-A!r4=*2C^kV$e7$9>U(Sfy;@fm5 zjU2_2260n@8W#Ut){sc;3o z_08MGn}ihgO$XbW<*+x4{;hLm>uC|sRs^NG=`LE|zSr47|50|ainCiRGdELQZT;h&E8MIayHsmv&5BcnJ0Q{V(VBIs zrSJf@Yt!0fv2@f+x?|J21as)<_FF&KQ}Zeng{^;_^X@E`!_Y%j0qIXm-o12~ngg|gKnf9R%etp0g`vyO=-|1~(;9LkidF2wn? zO}E>85hho$uTF6zHaIRv3} zl87x-wf|KjsnB|!9CtD95ed7^?~!s9-KTB`eoT69OZf9@e=)s&Q{8`X?@OlDQT|WO~BCjc1EoA>?Kbd*==5}6+1{th51aZ zx4E&>S?g!z(oo z!(&R0jK3|_*Ti7Ti;n$y7J%ZeO^Xl8{-Y+37&zNu#PFOC`OVFMwKw(z_ae72s%KyJeHmjSCfdtLQ0vkN60%I zCrq5D-C)GtcF6NQd*b}pEpcyzx|FA@7$eW$o!aTwHma?pP|CJLHbjaH`qMh3H=S{| zae~N$6iOlGtvP}>F|{kfGSUwHu~t4YZMnYl?lL{(tao!EQGzN-lN~>%k^EOF2;tEL z$0pkve<=0S?a=S`$`#SKiw`(-G(?pMZCWgT;O0^}>X)dH3cXM$AD#H7z!v)WtsU~` zO3zUytEz+%TYkv-nvW;NO(cgL%Jep3V>t>bxe`xwR@$@B!pX^Y( zNIbp#OS_)RD%Eo#7ioZOi2{6VUd(*^XGU~uk`GykjREWha z5K-K;hS1jzicmK z+Q`pta1IvztFP-DT|Te1mT_jHuF9rL=LGkos(eOnKRsF&t!}M*ZNoHok#i->4E*7$ zKXsOK<9tt`!MwYXJBOblI009$*E)YvgUeP?X*tS7Cx}66 zuWOI$SkK4w{E8{REW6!(rSb{#o|oenU8UWMt?~&LL-Ps#JY}G^YtP6AT-||(X>)B~ zDix;~OsEpMDhlfpUMLn-=bLHhNU`m<_b(7JW~8KC0nx0^6FLVmOny-f}hz4omc6k$1Q5Y)GRAJKGf;{jcgtotOL)*PAMJ`h>RU$)6nZ{vb zP9UR)f!^-REpqw2nHEM?aYhxU8b_FW#1W_PNw?cQkS}j~^IbK+HO_APSCJUwUJW%& zzn$m2&U3dxs9&Xd?3wADb#G25XkGq$+B1Y9e!j2EXM%fQ>jRc3pCu@P81lS}Ar7-k?mFQl`~s%zuK` zpv2}WSGJ2v^rUdlvmE8L)!VfSbT;smAR?5ZkuqSV$8xWihHI-Q1$%~bof5^AJ1H@~ zNZe1>Y9uJMF!c|0{$@)r*Efb~*LAeBfrrl!Y246m)gkddsRotB4AYV`vf<$Y>JNn$ zrbXvu!~Z;07clL}nNiEbRvA<&*HWfdcv5*|8IdM%+jlu6@1mnetck+KI3kr%A`3%*|dGb z0MDn~6Ibd34#FcRWl9T^vm64t{By=B5&tOlO_$T2yprEN{;|TLLV_xz13l+>lM4*0 zRi)eQcle3(2PW%Y3dRsPYnkk(2Zj#>HtZVnI%SmaOW2+v%M`MSV?On>?e;fhdYiIa zt(3z^NO?nusSJ!TpK*45=)6w3{{0JjFacw#Y@Y6%Azd;kl}*y^u(lV(Kc~0pO)ij9 zqA=Teh_M3_T;(2G=;R><*rwYGYY{0TNn@P{FkK}qW`kJ8DJ1?dCE6d%cTzcCsX~fs zS$242ptvI75B-VGP=v6z0%YPrV^bq`_H^%3XM~!Us-}*Szf17)!jy&TDjY`IVPy7M z+Hj@rP~ZXdD^EJ7Nt;+ol_1S@>~1G@%!eAf78|x?mNN_-9*~x(a0r{Y#aY6ciCj!* zaEyy|mL$#9eldilhn)a!qbuboMU{tRosn*)c=SP+6De}dT|X-+u02+)bgtm#VxXF~ zANJ84r-YvFA`f1vlB>&p)7NftbTj>Bl(&ctJ3p$_Xr|-O$_~$&3uH#hO=o0>1--Pz zv@YQc;UaE6rU*uC_@D}L73aqNP4fVlh!e}0dI8&5y~Ffux#rc@8V8an(ff!sg3VY& zy2|~W?9`HPD+(N@#M}xQ!6T6Tq2esv4(Gx34$tn<+QJyZeU;#7pzLXyY50SIKz~95 z!bTc26hMxaPDW_ zau-#!lnz{G={EO1-EOs!JA9WQr@5z0JjyQ-vyaF%H@>5nvZ$72cUVm#V@`)v(7+X&LD{;lMD!^N<9J(S-)c5O0y7~ zZr2OCuL0%BMHiy?0`F_;aNk%z(EkpT;b0 zkL*J4t*;zV8@l=;tlgpHu(R?%PwB%g+Ak4ZcFVZ7r*xcZrX9XJ;(Ym3R$a#y!TGTY zUPGd|3I2#d<798Sc-2Zd zjVxy{bgLYL2d95vsECL?K2to)6Ml;!%vUiM!D`@0-YE((M$WQ3^03u$4wGUF*B%dy zb6(}Rhq7s|YGNHzg*Z;y00j&JV(Srh70J0kCnQzyQsRzLvh!L;Yg?juY}+rVoHAD- z4I#ZkEFWR%E8SEZX?Kh|=WM_BJ^dxjQ|fo*&_Dz3Nbq0NUOKMbL@0w6gOYwj%Vh1yS&@5ZnV|$=SXi9pT=^e zPz9#7TfC8EOM>9Wn03?}b?t(>{-e{_LUD-HM-(?7i_T9SkOhx))XoNNmffjSq!`8O z`FfaYq@Bhki=Rlv@C4pBx^T;;-^K=dQsl4ganB-e-`ZK)4Tgt^^Y&*tTZ0KTuufOR zd;8^@i(4>83aLw4^5M&aX|8g1ry=LOEtyv)vHlUILaI|}j<>~Uo3*11kHS)KOQDag zxyqH3B9sjXP@0hH^cVSHIfPXP0j61Y=MHnc&HrhrqS#dF-0h%T#M!TMl+z_TU9rg9 z@)mtX)x{wedIvl=LLY2PP@YpIR7x2o#SpHE_r^c)rv5f)dvtD_EOW-4(*1`~?gJsJ zIMa&+RVCf-OkV4U9z3Q)%9NN*vfHbcY8;pVDS4UR7FXY-1{1FO_aF2IowRiX88;mF z6Q%#%s2g&Iil9!2~X{t2~_JZNq|lp3gv?4(#?ueqfGRg)2vO9vUXT4{ur*{GCo#cq2bA)y~!??7VH3 zNZ?bDWLhOv`tEVAA`uuEl=o3Tez_*_hjIgi{*#@ny4|L~K%m9he{A^{hh4bq4TP_K0 zGC(Bxt=3XJ%IcBr_Q>gswv?k_3b5RG=Ra(7#9=~RqI1Go=NO3=fNf%|^OhLr8~QQ+ ztBC)L$E$P1Hqx;8nb4*LM^>kEvtCItetF~o-v6|&A$R_KtFyxWSWpcUe!tW6K8F|b zFGlk5bDsBE{-M5!0ou{`-$NK-j)jP*$I-^+bP&Va<`WXwHt4Ldvfmn?r z^6Ex%#H9v`3|}ZljrmYRFW2hxCA@_y{Gk*?+o4OuD{Q1@!8A|nLXwpRcbw5-X)Km{ zQmo<>lB!aot4y0I+kSC_CWLFIK?@x9!nRVMU%DO19?n%fK)=g`6jIt`JI97<&$;$! zf57<65RoF@~6B2CB$8&atIZLtn3P zj8AYr;^R4=P6?YJ_Ue4+8!YC*x5hv`FaM2PsUw*x@VFM}Y=SE2s1M=R?andJ(Ym%D zIc>RftZb-)ZTj`dF@D|{26^BxA=SBZgg54#e(l#pa^&PC-tJ`PP-0;W4)p@+zBlw^ z*x4Fs)ft(x*4zm|EG2eVILVXB^#=z@|BLX$Zt)Q(W+-l2ROGd5z0*lq20`R#=wI%C zH-ljh^<6hqR$H5ynukN+2h-Xu!oy)frL8tlm)JtN{@FF!>KbWOvqI-x9@$YmbbLf{ zccl%7Yn3bcVW8g|d0Th;Q}r-L+o(x#vaNN=z$o(j9A_W>sgh!N1Xnv1BoFrkdYQ6E z)ZJUHjm3W|P$nFrmPLs1T+4bLXJ;#I zT#YnCBh5?nhP?ZY?t=^=X|gw%YXr(3`Km-0uI;VjhCj-wA7X#MV0}%VJpM327j-_- z`kLT}ptK!hcL|#*erH-8?(<6&ei_OPA`yOiva_kX`r zuYVX_f=4(Be4;6CtllLu&`G#p&iwRuriNb`~1YkRkiWHuAy?|d}4B?gS z&H?UuDyzdI-R|;UnpnXRycl?Rls%$@4~eTk4@gybE#Cn&XBgM_E)Zirb?FYw^ z?P8L7h}J|43>s%_ExIagBJENLd~AQSS0CCz=Bg6X|k5X*z@GiL=B*o{`GxCO)EfEEnHF zfT8&!M?ZbUxq3;YlA?a;cGvmUvVdca7V4L1d#tW?1lb$-uW5wQcf<>om;}XK9(M}y z5Gxa|6qNgAgUsNnTP{_Mwr2`tMhAUb&a~UEZEEG*ht0it_#ieYP2T31G^561Pv$$z z#y_QdDHx;vg*chv|EQLNUm#o#J$lbkab4j%dWR-RSNSSbHfsG~UGs=$zw%pph8%(} zMl1Jovg}Lx0;ODuCe$pE{W!D;Z1atZb`FUE9{5xQZT0B#h2l7`x#A?#ZAE`xD{f}X zm;Wlu1MJI4k;4myWxn}K^okT`IXCQ(;8uBujSwDRtb*XtRX%4)jqeL3Tq&PUu->X1 zp+CS`D)^&+8zCF5KcNR`#vb*;vU-6iC?BhR>y|hx8oz%*6O6^aohL%wkGL`N8vr~AwOps*X1q(>)It)wweE~I)Z1uKwTTBiD+)T! zVCEc@Q^4U*6~m)Tuyx9Wr0HS0lISUb`Aax{F(s@Bk?*dM?9#weCy1_OkIM6HFLWgx z%w+dfH&Y_JP%Mr6P0#qWJ-X&(ilv+l^w2y&I@0d?;$)dUQFm1~`2K^0A1SAG)jd=`0?wnu_i?xi`-3y=JvhhLOJ zY$a*V-HE2QN4M59MMpx4Ka_7M(JxSRWI+U57+~EdjgVarbkOZZGvzp@_FVR-7 z$KzP6=QoC5$}dK2@d9u1b@6pQe>YEr^BwCjiic*}z2QnRb8dh2MK=ro-8l|)inmJC zcuY&_-ha2q-=!D1jib6}$H`pI($?_^DH1m^K+7cxgS2uNszqC#MZ~x2mu`3OzF$a= zHc>+TAf!Z!o*c7K7Srl?AGTKXy!4z>_laA?H#Ad97<+UdbJUtfLNc;y7!$^PG1__h z{b|}jI4jHU{&A7aVV6fq(du`4@d|$imcDYTvXYwlbdcc~b#a><| zrbB>LNNOw?x>bJJAWl(bv9AGjFCXFTVHXg^RH-aGreUQxo}rIen0~$cuScCXxx|VD zGal>y?@4Fv@HKV7VtO19m8^|}(%2*BRYt*Gr`Lr;%&imTE#W70d&y9qi@CKx-a;Z{ zKU$?G)Y}G&IWWR`g7*na4D@t6W=_j@xYI1KxEj~NV{4cwVn<>2_IAm4ZTON zS~=yXPZS0dLG_t_=;saHIjC-3yT{N3ak#R)?vvDG*gF^%Tdlq z8N5M!$KkkqbTNdEX-+rytdrKZN5_LsH~Jue85})EP8KI8=`}WEakaHb5L(9+wjVZC9Nc48E06+j(J*0cr%5jg~q*u^TkPHLWR0@L~+;I^#}(m#7XuE6;o}op2J7URbF#18pcRDS}Bfjur2>0k9vgi z5#k8Si%m;5c-ygxuYyi9jX7KE6!lr`8-Kv`i?h~SoDflB0Y{ZDFIaEU<0>BE z+NQgTy!6TX<7t4AfUCtPAJ=tTJsT|%$2jDR3DplC^+M{4(cX|eeP!2_=*cYU8^6`s zFJ($>z5U7_Cn~*j$;D{-{L=Hz15&Wxpkf^pAm!fz@0|1Biz45CRL`-&)^sLCpft|z z`RHCL()DX!97;nm(|^R~L|Cid`^r)!v>RgiWG1d#HBz=6ZJ*=gQ& z5&At00~r}qVZss(Y|~2dHWSTU2%x3PS7+p{O_%$CL1;40yN>t8Kx$$vmZWBlhOZH6 zKrdIG>(!!>T)N=6Rt~FCsQ97N=mq{(q)9amGOr%vLQ4& zB36_5TS-AVn6{1(qe++>Os9nRAh6t#eOIv<3*ic-ZLcne#F1U^>s84z1-)0B04wFq zxtgigrI)Of*KRA*YZo(giF#5Z&RSDB_?VCP4G(VnwC1d}G9lp7kuuq7%^`3!b&+=1 z>U3kx0P`6W_Ue`5bYp?23Ie6T5xCN6Ko*G*pBs8{!LWejYG`VLuGf_@)@aV+=MmZ< zMasy@RtCo!m4Wy{SiM!w$T+BPE$ZW9y>2AIEJ-xh!-FbFnM^*&|2(Cm5VZeG{X0U% zb2rY>bJHMQ<(9415RO>6IFRz(WEuI;KUx>Vbmal@97uNool-D;e!qBrbGm8?rvK>m zOiSVA@O8ck2Y5VvR(Nl^OLtp_$BTuseX9w2C~1HdkwJzOT`*Lo((T^OxnIe0y3)l+ zc`I8y%q?FJ;ebsL`$L?(-hHu8SLyXvn9OodK^vz3P%!U2ZpZL-xltB4DVCiwJD>%G;R*~fn%4d(z`>cbqyB{@~NVM z^=7bnEf<7Ti5gvkbJ7tlZmPQ(+dJZ{;2A)DR>4$h6W~mASFYd#(|UwND3GVLb%sY&SKS?|0*Nfc5n6yI=<0Q z#h5Y@CDOaU(;4%Zq025+B`&&DP8a`J ze!g&s3k(pCkc*N3nsA5Iav{-^{$s8@cvv2a;RRkM_lF{*^;R_ z7+h`p9(D#kqPIAW3F8Ls7bRriR_Yrb^$5drMG2tzujx?YQV)pd$%g_`We=d&lX6?I zY{`?T3Ws5O(;ng3_l7o*vB%@1oEA(pOD+N%S?C0q`M%5a;nU6#H}k?S6%KJ`(tI&F zqaC+#o@mEDAq-Otjyo2KkdeQu`sVW*cV~<@_=yj-#i~?K7nh&r454MCt#M`C2NT5W zF*-AtiJ&+(x%yvvek^YMUZ;QG64fE8pKZroh!ZC&?y76Zan(CTCDTa$>++~qg5F^= z0|E4K!{Mj1*3BeoQZ@w(CgcHwvpnuwiaoGemffd&BYADwSNZ~~A+*d*9ZSnDM+#P> zMEk(ht1+Q6a)xq4pJka&dsgxk!B6v8x6kRoWCQ~lV11&NJCUq>s}c~??LM67?!YVV zKsQ$JbNh5Tk0p7f3ngHBUA!E|sc;Wp5B+KEb9<3<6In(SQw_b(n0;dZngtdjO*8E? zXMw!-p&5Ejx< zv@?lK(2|P=;@InB+?-ZCjQZ6Jw2j^RU71Z48(s;6!u{$e7t#Y%{5TXba`Xg zJ$h&j(vfzb(>&a``##0Pe1ZA~otAgPNaC-f6=P0|L9X2 zDYkK7NZErQMrQkI57%z6mI=$gcg_<#dJpr(X}aB)z}*q@gN9BO)Mv=D#`#CVc5%$f z5jzGq*Mu@{kMqmD?Q3W0vtQa`edh&=SIUMektVeJ&R-~AS@?}6fD|0jbL5kvTt^U$ zv+Ta}CyQ6gPiTS`+n0-dubilR_}%W?b-D~Lj?z8ME?s3nt<3I|tj`zvtN7?^hsoPs z3fIuF8U#+czFhjKPkjhUljZoUbjoE_3PMVSoKVtP$CxJ6+a&MH;@=x=9$z)3FCqD-e1**p~wqsuc3oL-FjR24%g2=iP=J`}3B0TUpc zk*rLp62^0D4|}iTqO@WPIK=YeMJ$WMHNKBU-y;*`B+><0v|EGYy;$*il{TNN%a;{0 zxrKSm1;%LW?Dn=}7f|siSBToU#{EwV;}>4d7%^T?jsuluNFT>=SY1QSapummNcWge7f}Gvz9~J z_aM~wD8KXzh>_E0>+2A%#lG4jZwt|l(0up%E{~J9H?29E?3+EWtjGB887BCxbnkM+JyartI2m^F6EUei}edRD;{T|`y%vy zZRUu_Z#$&br$oA4A3FCdH}NG@%2ACH{SS%9nKcHXT$Kja_l($eSDU)9?ALF**kux= z7z6bib57pLIt*?wU0=W9i$o$L1C=oaxBwelD2J0dUoq9L>^CA)lyba*KNMrcX6==2 z`W?{9!G|gn^F=A=r7(^w<+f^3${MVyVk~xdttjF;Lydt>m3uQqH@+_lhbi$uq~|?O z{QE&zuhuU5EsYo5IVGQA0vE!PND;$jCyZ<&sNeEJv59+p${r0@aZ6 zM1(VuaPYvC67X0P>rCVt0TN8}=*Pl;R_8t1w=ky4`pKdjpI3@2a1gi$`~};Spj4va zQL;g--QPtMjFfld#bT~8Dx|=t$~TcR^Xq%Ga^WhH>+|^Hh;vQQ!+IKNN>nZsyIHEJ z0eYE+UU67N^W>${#Q-}^k~h9UilBkTlc4Rc1-;@DcphW$Z)sMp@^utm?(F!f`PMabY%n#@n6j-NYbJ<*DtmF=Gd14}O#=-YFZ;T{`!@)7`c1XuCk$$x zsQpr-d5p=FIoFvNrDHUXhYyM`++3o zzm(y*63v4gRqcY!abzQw%#%MR_EW}D3W78}*@@sH8yJj{`%kZwf~y%-SWHXlpA)g~ zQLZ5=BO`?>^$6L)`&O8<3MPX3-y0$1U$cEJiYgBZ@e&u66fpQeSUO64FyZaG!0lg8 z+ZWEsiPLXUD@N*vkvElEpPOrprg`+wKIGX<=jic`a{9l%K~5-WbqRsPlcPCqT7)a} zRXFtjWRx7!z`X2~s{*F~%W0yB8{GV1BB=lM1>ytJ(dAKvA5}gOa>CdfbrhCp9Pk*y zw-V!+HrL6 zR1Tvjp-Z`HQ3J>lkh&s8TMVngbOujoxL-m)&2)OH=Rx-Of$dssc99&*qAWL`)c(9; zxS9-H^d+xRr1 zi-Hf~g$U1icg@jAd1stl@xo`i1f?j!*^Qgm-J{z}khTY@TW{mV&p6-G-he&l#0icB zP~VlZuGs0rM;r;J-6r&lk$!DY)va6&_^8(NCy6FOsj6U)iVY(5!8>$EU!##8*zG*U zF^gU3Vt{>f!Lhi3iTPaUReQvlKjvLx^dvxf@agOGuOesw>7?%Ne3!ytkPfJAWF6t8>};Pa6B?FUU-B-G(j`MxNeDQ=dAdX$L4bqM ze6+lqEv#~wdX!BQ8pL_}c6w5Wl!jivO29m4W1q$P&|;LX(upZ?o^DiByp1IF6tRM;kr@c0Pi&h!J|w=&psr+os~~k6CeI3MWJq4lEB3a&~>xYpV`^Ls(EGCm#Ax$5@Ciaeu7%@X@o{ zl>np4{RhPXCK7oU{qjhROu0VG=gKK|0^f!=nP-F2w5WuIXT+QAi+Ox)KY@fo-v*fX z)iJ>syFTHbJ)+`lwl9*g*ov*ZKcF4_*QI@I&ho<=D}Ta%b}ZLtFMv`Z02(z6tGw63$y*+B_1rjT0Y|2n>t6 zQAmF656O6dwcJMd>=~gh2`KfA33u!h|Jit~ZVZ#Kb+q`1%Yo&-a2Pl$K}_OyEf`!L zd)JB@j?mQjdYA++_SkBcvakutIljEmpg?~GgVJ#RyXR|V<4lghAdJm z90`i)z_4@s@_9XMDc!hMe_)r%;$McN{MQ8Tz^)vBA)Pj$RJD=z!0wHl$JwDlq#~r- z18**nE4bHK?o)1HWQ_A5mwWPkUCzJ>2j#pcpH(w)Rf;Ovlf`vpUVw*LnHYF)rnC0P zTlA6|B>*;mvh!5iXLYY_2ut>h$Dhj5210#!W*d|W96Yu)O09J#t47MU{L~94Zr5$UOtq-`C3atslA}EQ z*NplG?w%!{U{F&h<*jJ3mL)aUFYtJCp|j|pQF^FABtqb7#(=2TE%Z=db_i^vZ{T~o z{vy(Yk@8mDKyede13%dyZ++rJ6{E^Gwmqmpp?Hp7N>xD;y0D7lNwiuzH)e~);M zORQ+Bng=ER8*5GGNPwymuEY)sYAH5zM3euT5`!89i%qXxS0}cfltE-$`#1KHx;?Ey z{-aW>r)^VG4AMc(_lg4MrUgWDxt4M{jzR3IQPJQUCTs^nFL_^0&@aT9t^(;?-@`oCrAtiFV*tEQZ);3%N z>zQda<~p{*VF=Ns;yE+DH=V(tp376ecei{EumP1Kma97oDMm)hpkzn+X>)_dWQrGgGlk;nhbyl? zDMxZ75zo88ZWiJ>cc@^1P2J)g=HuxPuySqFDW~OCIBFN20YrdJ^Ap8f-av}_VULLw zVw0Ow#UPztsY}!W|={Dsam&RwtxVmjDC^ACw)_quJK z3c5jg=VZz$pSbdOFKWk>;oYq51lW#*BSdbEXskn-XPQOVKB7%Wz%J;!FsFWg54l(HDmcPw{) z;n^zwYnu9?dftCU$fa3DN~Gze2hHUm#zSUZ3iwDV%aKd(eOd=hzLxU(BH5dy0Lp~n z8pr-3vBph@YwW=dh&b*@6i@>A@HlYD8$}8xg#+Ix0tpQb4>w>AHgawupEL$`wLLY@ ziX$P}Y~S>L5INoA9-5&`7#`K-l=G&&N$6#4OL$76M43Ir3%sii^q;)v5vD}%3??%9A4WgJAHqfR^Q-wZ>Jpj z>v-!c{9*_^IrSVXSLmUf1ygicXT7egK!osJn9QxwI|;5E&ie@u`7RXFT?$8HvqIVW zx#gDKQk%B9*Tag?B|!u0xKsSV(MUuZBPVtV7C&&R z6CS1!CiW0c8}`;HZd{qz`>d10QQ~rdX#*wpA0hgDQKXqdL`q^oY6+VuOMRi8XeU^e zoDB8&qA+n_tjuC59I>iX0{Yl$+4rMQ^&+GJcKvqoJv(PH3|TM|OqU)Ox4Y@+s0QI? zo`_}US%Q>2<;wbmoA$_20UdooXWNMrcFL=cWNE9HX^TB_(izE}0Hqu_V0w49xXwN1 zX>_@Bf^4(QydpvsJU4fXRYjVw^{8)fupx8Dlqh}l%|IW2R?Hwph60wak=BEL?>Uj_ zZVZ_cE3)N{5&nAOZKM>8mUnPEj6d86+E^gIoBX&YAR44E##wKIG#eb@n!wvWInB@N zrfvwkj>tQUx+!g4&%F^N@7xsavj-M?MF`0aZVx3gG}4!MI!W9+$pyM53|St$Zy6t8 zM7ja<{y5qAqZGZCLkW2B4B+!bAyw202f*H{l@D_1s{A6b&kl((goDy0UrRY2A;R6G zURp}x4~N7}cTChnC*$lxN9E%4-&+bg)nHl`FDl8|r*N1m^$4dptxqZ!io1bZk159r z_L1H~0pkbsA12FSwx5xrE=}QEwp+@0hqns#jVmt{ip+@_IzbymaQuEj4&$Is2_-x_ zEPo4@7Ms%wn65IUQ?j?uRVjM?BVR!u@?e3tFG<~1m2hRCAwg?pDrcSMQ$_P2nWB97 zp80kAghP0P>RJv^T|pJVQDyuR>npyMQP^XbYPUl;NB;{qmlPpSm*{!aNoQUO2DeJb zmU0;DLHySxbS3ROpH~lrT&>IH>JNz>A^qJO5ZcH?Xr?J&w$REMT?VGf6vw2}H`vO+VTVGRoPY*$D@`)&|-)5jh8EazDc~oP@zhf=$k87xaZg~ z<4Ai51J=1Lh5J**XghSTe4yCmW%1M6jy@?KCKojSRT`(;L$a%#NS0hnr~(fN51zCR zyXnOZj~qWa_`U#(5^6F5uzFQ)J}KJ|?Ij*2PHVK0d2p=ySgZyGWWb{ zKyEk`owIgg50IKZdB`)d;umIGs)X^&YimXB_E5cYRiKf+eZ*SNtwpN1&fc~~F8=sz z-F@UCrDv=+*f><)Hzs^%i%dx#Bi8WEuF`HeQeeHzrCuCu8qkm<+pYcPz_RPguZyj> zCY7sCyh4kSj~9zOnorX0rA&1OLw;`L*la$cx~uKU3)b70u1eM~QGMFl$+09bxME35 zlfm_4i&mMSeS0>=deJ@djv2A~e-~ID-aAp+8hiYAr?s6g5Q@1mcA}DJX3d5Sr2uxe zRGcMSV>!SK;jg{cOaFYMUFp%`a4ud{lYNd7Dw>gUK0+L0XBB&xCYIEcgzQYb(bR-x zQllf*OC(lQB{YttfEe-P@P>L6JX=9e>afyDX2z#Vpi7pW)IQL9nZ6LHrejMAKV-ei zBGY=G#gfQ^_ak`&0i}NMXt703ySbole45m2j+1N-#~D{9#hn&Ea(h8lF*p*6tk+09 z0aG)3P3pN^CNWzJ0!NwZl#+N}KB>txUvG&Nxzf6WgOBrlgYTpPTjjKy_FPh>%^bOw zv85kU%2g9f8oSUcV0sTj(;AY-9M}V@-1gdO&7(%!^mg*mvV{dG zSGgH2t(VD@=K-lnG}4>H6i71ELkX=1P5jl(>Jb z`1L8B#oP2_No0;YnbpeoiMH*e$0DuoNoiMwfecbQgCAZB2Zr18Cq1`N+{^r>fKJJX z5uw47R-tjiQ zRMID!3vK-4xdQalRh5qF@@c>N>C#jI2#8;ut2=|2ttPbJxXfcS4evJ1`Q%@=hmFup% zR=Gh%^GG{s=OMXxN230Ec(ii#V-C3(GXREIke2VZ&d{+@#TDxfkhihdSOX5KAcZqm zPcsMdyC|ie)lVi{Ue>i$m2wU2WWHGPp{3)4YDhU>Y<4+D%5Nr zIwUCHQsQ!^NfnFalc%QY2r|kAwUoEbTB!VLJ}1eYL#-csp3^&cMas9y1EQ^;cxnMq zhH28}VVN9rm^c0YP!VJy8tBPetzVe(pqOz~G8w3bvSgA+c;h`vWLA1313%TH;20IS z$iezu*=yP`9OU4025E5X`8?~uf2El#^(^mZ}d zy)r`;c(5lP<)+A}(aoctE3+4hRZ9`SPh&A-c;k1U6DL&Q)n$@->dT1H@H?l+xCtnNA%i2fs5*o6uC5 zTOoR|4F+t}p(I}$D<7&bDX9#Ptg|w4@)6}E70K9xt)pY4u+LzWV`-)fMvGOiZ&1@F zMFiNKmeRl90g*+L$9P@h!5DEL>$nhHU6y5vv7d(M@j6nd^4J#7?>viG1`a=6WmTN# zckZjN$1kXlO&=)`53#zTN|-A7X<~G@+jO%1H+8IYhj%1Xz!0SD(h}0>k92kkCw7~ zzX;Dhp-pI%tJi<*jTLvHe%3{?^|WPAYaYX%IK+Y$=w$~zZ>=)d2MGt1JFwf@%Z^ou zZ$&Y=ELOB*gpz}D<+=fV5-&zMGt@BBbtT)^4e4_DuiKE1M>q{RDVsrOzz-?MT8jRA zGWB{P)gb)mto0#VM@VsTe3LBFIQ*n|*eD6^H1yMVVKed z$CIa~lDShL`FqXpOkP@yuH?)7Xb%;A> zU@(3eI_Sc_^Ne7X8~ja`Gvf7Cn>A82#vXrccYbwuc?^%?hpj1`yNCe?)nJ;EFS3qz z)2$NG>Gn{|Pu$FMSPtqVniBELWdi9OmZDtA|F+n5xz3v`{9i!{l%vXZ8$?5{GUXb3 zz?8>Z{n!*%(AR5Sa*B0k4vPB-nbyDWeWe<)X{JMS&WRibSC~&Rg~!ATB8X&BvmqE9 z(`H(KPR!69ERqmA>ySW#=t{TMSXY^SR$O8xXNgY*hSIeMm|Y1h@dQOvDI@#y7)(z@Mxw5OGGwzy@2ilduD<4$Tt;r zW2vF!gUus{NP_P$$IunEB97*v#O7%pY;d=Bd1Po44oP&9SyDrSiS?muw8xO^MnMN0 z5%f;JNaJ}|iYvw--4rdZH0MuMz(9Cyr??{Z4P^_pmIL9vwT>TYc@e4FKNfp^k7zULTANZ041K6lvPj0P9;lz*G{E+r7K!fdCqvk`T{Hl?(uFGU`0!|IC)OR1 zqQJr9dRNahBZm{M*b4&^DRh^%#*hsi`Aaf zC5HZUM&xo^5M7kRslsOc5|$bGuW5wC*aRKGhY#4sV#C_3^qloStb3Rst^Vb}hBZ$R z*KxGE3?A@->8U_(vgchI$aq40Vd2NU}AKMj0M(T>C=>N&G&X#J;JC2>rYNo!qBwrVL6E+kZehSGF|Yn z8?!}o9va1KrXL$NH`((O$#0Z$5Ye`ta?`U#8}2p2)O=rt%}x+^)6pwuS7~7P1?U4* zC6qAg-?K=Z9g?a0LWn50sMb2iGQ)9n!DxfdgIO@#9J{g86L@p5vEpr!-|E;armI zX4p@A#bJ(kD{V~`Mg~XHFVu}bhp}>XgtgBM6JtnWPWbJ%#iBPx&R6XvWe9h8t8{!B zBc~)y7T?tN)VknXs$3pHDLnR5$k-BFn4W*w_Xou%Y(OYiCMYAPw3sE1a8|*!2Lv{k z5=n}P9;zC;w5<@IR-G63(>(mO`j_(8n^f?=4SStlH!qhKO-e7Xf zn1-AZT`2~VSl|k*9;8cjTObk$%L24DghAs(=Ekc;7&gs!)wfRZ-pLFT$uJ#8RzTV< zQrsx~b-7%#@RBUeDR*q}_T|`X6uMw}s@#$5ZP~`8f-+3WI2|3M+lm=Ur%c{!J<0() z>}i^5%E4sI!=R=Nu^AdiPO%8%YPo_QVwi3aDdfz+TIz#=l-WzHtsit$)-s`;vSe23 zJA81!p!yJO`tVNKgu{aT;o4($tr*K>WEUm!RN$sO9x6_;8kpc4C#O6cZ><>6T8}P_ zl;^ity|^2xc=+imJ7!v6bCRjV*RoS?JiMGH;><`tR zt&Cyj^LuA{@D!)a z(xYHXQ6+h+x9h1DTDd3<{gm)V|9+n~a=0!LS>Ww<<#r|3bo9gfRCv3#Oj1luoDN?$ z(@R=A-8>AiekVOIGS$d4y~9X5Sr zdpcvFX@24m*P+UAwvn=`&>OufSeM9E6RY7@gvyZxMH+`;IypghV`hQZl#u|&u4s@o zQV&Oo5eOXf9L=Dn$`_}@7iEj&@&0 zzu=ALB3u4NV2_TrelPx0k1aLQ)GHQQHu)WLOy>v>QXq5>>#KFyr7iY!n5U9#jr`%N z|6qY^IO-)WHc$0I!wYLY-!eW`IE-?eD)+t2nJXnWLj!v*&Uzr*T*zIozH`HuMaYJ| zwu;%fLE{*{Y0f*vT$!;jG9|#Vr9ez2vmgvjBOJbUiI~D7LK0H2DJ3}lvytwz24H}L zu)C4fn-8BlGf|*g)bKYCiA3|-jSGxyYl%#Xf&fMpO1yDET-)hM#Z*ax2aB?4WGUjm zF7$c?r%x@4@uI*33p(d-eLIXrTe;bZLj8{#lD=CTFWHh3Y;CKrEHre zC#*5YfsELCggu#Z0wIN_<~x@fxY8T*uX*v)K!5*&xBFdf^i13sxiZe%eYQ^2;6~va z1#-Ome3mJ3Vyj5XYNfRe(H0}<)_mPp#~ARj$B%Qw5FYyhx*FJ|+o_>Dz0p4}*2Awj zrETg!6Jw}u43s+hq_-;<*r{*6*~7VKu{fr)9`)%e!@q1KZ$EaoZnp-V7OMSwKeeYH?WKqlMI0{`Z54zMz)%%*UDa?Oe6ZR)50Mi78)F_u3Aj%u;)+N?! zZSUUdIl*2udMGSFf1=nM^LeR`WK(5oBk?smc*5#mU9Xv%a zRaTUW-gl^w>R8X$eqmke={GzUB2^Viyu@)2PE;vExVBjNWZ7cc6x9+GffBzJS}Qp8 zoIwO>T~dGDCnoUC=CqvZ{y_Ry!oDxun;4Wzx2=|=Jma`{5l59MUE)vX`|N5cVEkZu zpin-3V{biTLo^0zbyDZlQl;SW{|GzxxT>o3 z|5KJ0sYzmLmM9vAk=a;z$;)1Aa})(bIijMdc%!`F6-6@+;-=Ag@7Jw_A(b`8O~%S0xOOm!}37@}x2tR28H=6D5vu79W&m*kk=O+aboT z@!Ww9j-QpQYO!y{Ib&I*Ry;5=JT}COG8U@T+~cBl_O1wLOoD!8*L*CjcWXIgIg?j* z)j%7Mi_hmw*M8}sL#hU0+iCGRx8iY<8Gc#Y(#7Ybg8{aO^rr;3H7Rx!Q+_^MgcZ82 zSI^c=sRl47B3Q;2#m4|!q>WsDIrt&V`dfjHKD}*gagOx!2Nw5qc|S(9W5iY$@AHEP zi&X1LCj~>#m2MsJ&BX1hxQAo!7TNmAz5;s)p}oI1OWs5-dCx$2?oh6%;#ua}SKG@s z#ZGOqYAC-LXV)-LX}awG${}3>1Ht2B2d9n6Ijp)av?XEP-3;Ir4{YMMMP#|#y8HD? zib6}3&nv{|q{2?ZK>Yp}p(00ozHFP;#gw2$2`^DT3R2~KmiRkE1wT@@Diex&Q=Ng_ zCsa%|Fj4P-Gmv2q7(7Rn*OEoPDD_G647BK+Txs$rs~!R#4_$WKK3}1mX+RtKM7T&V zh*Atq716l36vOCQl96)TLGciMGR4jJR&?Cy%*1O*QNXarLtUKK+?~ZZ28U>ORLm{@ zR!f0LZ(GEdI}N?t*Hi;X&q${shZYqN7(?i_(`obFC<|Rw1Au9tG^ZhVR?$`ggYd{^ z(c#sh;uh>-c%+|kmT)GI{~9BUUNKH&^S!FL3Vy027hB6&yFh|hBB9iY;oN6lD3Zmj zEVnrenK333sUk+ZQG?9W-|zw0E2rJ4apr_srgp=qG^Yv2PJ*GZ6ay1`JV&fg2bd`_ z{G1ie`LQ%(BzR;dI^pymTTOf*g=c5NIf+naX5tWfh0BI)#x^xWrxaUExfJ~PhqCIv zu%-!fGo8BhK^GB=w*SkfGb2T_!KFGZt2Fe2fO9iz7yO~%Q)NN26VB_DILY&VNt|;J zo>S0w>wwwXUgu>|RVuX`c#(}m1G+y&AE0l`abihQ4Gz!y>1SNY_%sXy5~e_{9^uGC>@pWuu2jST#p{_S8SLat3L^aZ$-xNreL*gw&hqPAj%J z`LAd7oks<0(%Q(GTH6B^PA`_^tCia^UVJmZmG1cs)5A@i-lZ`*3Mqv>+$-Wb?g2N% zruYWJduPOTw>_vGRigGh#oJfJ=WGk2m|~3jH3&zttj?p%t8Tax9@|r7mlkGc1Rjde zOb$qYNB2@j?9XQ$i|qx9dt$%3AZk^Fhk9Jwmi4+gXT6)`h=#`rA--TZs&HU6aGX3Q zzF=Y%Mra!X7laB~f<}VZ#K;;M%aElKDG)K?8yl@yF7;whRjs!zJ05b|wBJ%~LfNUA z+eWlfP0aXK#$*PM;Rde?I4Xo3M`8g#9X;^vys9LqAG$mk z>)ga(MBPmYDqSQGOH`b;s~a+DUYwyOiI@3`$tWQzxE)7*6(2CQE1Wnw6#g)7kVBK5 zp)6IPm{+B5EoaD|S6Eb0>QklNUh(qGD$#;r&r8a;vfV~enN?Q(DUkPV6DPT&sVv5i z5<{XL(rbmh507C7oQ=EyQkM>8xAMMxEZjMk`i-_JK?h-4ksDR_5gn(1gU1~QW&d{P zl(S7}%OxG%&g65<0~ka~^gZr&<~bmMc}5;H$4R{QsRc<&DsjJgvp!5cttcOgk^+?Q z@T$zIaQe|2$_y|Cy^i?&_fuNRMU8aS74ZcN|DJLn9TiP-=ofSq_&^vFFTSDy@rNn} zfzRsaKfSNjhp{3Y6`!AMt@JQ4Ci5-@A0xSIZ4*I=&AZg&5T~-#cxGuf&=dae{WM zbTL)d2yu!GcJT1LzlH*Pev0tyz#myEcxfl!v&kc-FJ|#7l($+A2FJ7GVMtu-q>YSCg zOnXjmf|hBqCyzO!NOu|ln5ptZs*}Nfb`=3*O8h-ioWN0@w(#H}*U6VMbpL^92w#+o zQA{-`?zN~Rh2jK@e;zPOFgs~P&d?wx+^s{qGD4gnTO%Spdwg_7Msisp^J<)Oko?3} zoyW*CGN-~%us){z!VjijH4!IBTcmgx-+sMXobU!D#$x2kI#K;v7A8DLXTTBx#g%eC z+rBf_$zCLNi^@~pw;N3j9^lZTGGRa&$Y$587c+~VbQvvld|WvoPVvG6#nlfUH=GeC zzHXsI3dR8Y*9CEsMf)(*UBKg}W6Fej{sNIIt?LIr2Zt&F4oduyhaG#?8BdN~Fc{_R zn2kY~eGf!>RcbJe%?%cN=Phv-Olwt$3fe8Ycy0f`r)6a6o2tGSr~fUMkq4J*0u}>U z1Q*!JmjkeF23W%m;uKdiszMFOCW+T6?jvb%oxavfNK`}H!5y7nY3eXk8bj!wx$g^}(1C{uOFLtWbBU&|bO~syx6KwO%t-{L zF~AIF4LU^Qu?D|!OZXQPL%*f!18Q$42ue9cNE6?2?*!whYKR?q zRvaghASgZMmK|^=^Fqu;ni!?1viMN&60hxfQXr7>@C%+bQ#cHdC0oRo#26TqJ1LQO zNqpI5Z74n<%$+JudfTLwKy0wkxsE$DibtlF@=A_4$$%s&gd_gOsqWn`*m}%wO7P(6 zN&G^6#TcgB!(?w~kG7aW`tLbT$6pRuVb}x4l&C@YJ=*C+?sU%{K0E%p1gGOM^E?ua zsq*1yagzLWC}x_6{pm)L3G@<72+?oH-xlrs{P4z5Xrs%!U1Z;$M>TZqm-t(Sb2o`e zI7??1cjB#%Zr2OXX%iYA^^OO3vF11%N5v_z@saVtpXn`?az>Z)VdA8>`G{zM{gNt9 zF=-B>T$si&w8HgO{!?tCQ!-4Szu@+KFJARfiVsXj#JKH-9oJ0Zfwrjuw_ch#GNPHr z-yZ4KC2c?@${Fa76a@dnf*BG#LeKWj^(?6cA!uOn4U@&0NA^|wHa_#H+b+!H8#g=> z!h-)`ac4RaWN?t%_RR06LTwHlPhD|uXU_r*#{2P0wz#*C_GzZNioI&JT>Xzu+NMpF z`tb!(&W|_W5eiFqhzMENJF*HmO7u+?Uofp!O!HJ|$7jbnVH};E4Pm!xFYy!h$s5Q? zon&GR0$01fJW;F-6wnr{L8t*eCtb$;^Yv?Z49S)wy~LXa>7gUeDK0DUhZUxQameP3 zmk-tuGLOoEj6e9p^o8*UGo5$6S3`BI$Gb&hh<9EaKIn3wKoRs{qD0$+H>ZPM+V?yy zh4b29bAgOMOq|v<;iztMAV&dllKR1v{2v3o1tP=rmu!3 z2-6Na?_hOWf}DkIPK7|QBKRp_(k%L#WDlFo8B>VpFw;a6lI zX7l`EOz0auzQ?C*3^lxJplLhAH(ZB;p^0Q48+NI`*HIWukL%lBDsL+stxKQ}V(&?j z?O3+0R*s#bAG~8pRHMXer^F8o;gax+F%Sp*&vN8l0fR`1cPeCL*BkX&$nUo%Nk3*)>-tG=z@f zZXDa3E$A(4D|_?|Pjl~kWO{YL^h`SA#xa5gl+m_l(rWiU(yx_Z4}(rA-j#D=6Svy~L zA`iR*X4KCpbvAICvPEc2*Rw7OxY)6JE2LacPA~t*oZL~N36JH8?!9EN^M{tgdk##r z9B;1xZ7DJ3nA6Y8*p{h*wK*-jdJ#kkpwHgpEa2jWQm#ZZWgRAuP8xz|sRO?MV@K$kx}rf#$|&t%jwBiNpA<^&5^L8FRC7r_L1@+t8& ziS8|vv^}2NA%5Y4q0%;}OEBdr_43rz)g>x(odpdCtMx+l1IPI=C!awt^_3(D7dJZ# zz3XyXO0S!aIbU(n0EVg*B{%@Fz*`_{s%?5U!|Fu(#y}`=do@aR{z(<3zA14!Rs8UH zGu;|6qu=h;y}eVwOIOchLd2rW&++n}mX)G6F;=BZa6tE0PVDlBX=1(Z3W(nr7;S}6 zqQv&tCeRM1mRA^y)v_a@cPGn1W6fmgPQu&7^SC>h~&%bL)Nb zjqZaqj$W(gxb@!uRXNExs@MAQZoP4fbqND380PkPe!2ccBMGJ)fBB7(qXK56Y)W)m zd0!x7!ndQH?S0Q%-=bWZ#!;h0`BW!?6e@Y9z4+{2+ls9#q^3ZEX+XVdl=!s0lf|{0 zG}JfH*B6L?d2@Ab^wJ1}5)b2?!0F$Qwwg3`+HbFreJ&$CS95z(bO^($NE3@j;g! zIpS7QVnXo5PK|IrpO&UisuZghmB2Iiw{d0vW`ZtGIVzz=n2h7}@os>j93=*4b$^8s z8=yQM*}MTk5Bc}E+ejckB@e8IhLcpJ?W#z^VZOlC5)u0%OgVqdJ2 zoYh53fe5e~1Tx0nasCGth3Z@PKX~-MZjXCiM(a@T0FPjn7{UEpr5tvp$~)slMdd%W z6hq*y#y_uJ|48_MS2`Y>A9(B-dw7_hp zglWR9r=7MeA<;}tC!KJZEHsSyRTB%S!0r7|z_~E9N;eOr`0Uc8BJb3FkYCf72Q$uBuUG_&GPO!8}o3vQBK)V#9Zc5ng&vs!`&uT23pNg2OZN zzstq1Rr-{y0oJ?uQ8AJ=0|ksyN^DjzpK(wg&NjgO`<%bfC-a8_M~O-2oF8{|)k_ss zy2`^x#ZZ#Hs1i+;=X!Sv6C?R(@rNg6O^jP-_6+?sg$E@bJ}M)4=v;|K5==#cjG$F| zVikcAN>F3)qVTNRt~z7Y#+8atrmOsAv=zyaRzc?nkHS6jR(fsz>)E5~sB<Ye6Ja3b7xam0?lE z*SO(A9kJXyWn!vC#97@hK3N?OeIDKD-oQr-re=|;&(O`HJxfS&K!*s^xAsW?4f+GD zfT4cbE;-DbiDxR$^@-gh9tIS8nE2?^AWVjFQ@CL$=xP#LCCf18`@lB7?bG&@NE}(I zW6UrO-yyraI#k<38>mlqeWz&e*P<4Z@|3oH28D}97>))IO9`LdXF$AIVqS4Kp6heR zDcOZ{^b}W&De*_;`V1)&UAdV84)vo1nJ2rN@0|I}_ZcDdH3H0M{L&}mn0SO6?byPE zRG;)+V$u3!?H2=V;w~|u`ZW`fzLepdBCqvDpl{U{Ta;g_G8`&`Q7#Z$den2aaTI3* z2Qs0l3WY--5^%}#^Gc*=vFYQ*`=Ush zM>@-hv!O$J5RRTH{X3Vx5NQIY&#@h)pLy$+M!7(&?6UKmg~e58fYo0P$XmQ6S0urs ze}Wv~CGpa_B!(51Rx%ac8p4rSKhden>5eLNp#(fK^W`{hP;EA&Pk1aymQU0z*T-Kd z<+l?j?RBm7YxDsbzY6Wd`bWiZZ~Hn=g;e5Q2dwY7bOk7769`O3zGEg1Y-L8Do%rA` z>-3Z@)r~OGx2<#`hiqztqfFT&@sSGa+m69%MRIHb_Aauf+S|?u}>75edWeZ)xt0JSe4bU^RHW6fbjr`&toRvkw#u^&g4hr{o;(H1<|ykEEAYyB#NVt&X6?f6sAm)~5qzer~ zR;R>}Z1FirJ6yHcic8i?HZhb5jgkLV;9QB0R*lfOA?Zw^)A39*-3WmX4}TLmX39{F z)Bt}O9)@t~UD?i&$dJ;$Ow42=mBzm1TqS;nS{(gz;@N)})JN$1BkCHk@{d@k`t zJwgF`Z@VuU${J47>G@3W(APQZS}%9eiwF%BkG^ekTrqq~^(I%}wmW1qrn6u$uI#&X zv$X`z2SUpA&FUh4BIzCGGf~)=w2AdMKBEU{EtJLju01Ax^pftP4^m3v#EGei_tv7bQOC3bI(QVR!J3jj4zVI$;7!8DS5iYq>XYo7p73%)ApSX;z!=p!9ePV zlw;ZAytj~P2poA|Jb1l^u6aCqFw>oQ=8`zi(Y4KCVafKR&Bj}G_~2aS@hC&MAI(<0 z3Cl-&C&-vuyLGHLRn6nk0U0vp*uDCiiD6oK*16c-WPs3OAN_2LbJ5&uG)N!)Zin+L zqrNJ1DO1Wly0@9g=Rm3=1eJ1+`p3&*u|1Uf%If&`FK6YZ?@jVws`%_ji!0>)tQjb) z8yssd$@@7{s2qin{C0B8UgzSZbJ}ydL~?^m)|1TUr}N=6RZ5P?PLm?lL_%pym4~97 zpPS6oUk{_p`HoUOCx#RcdQC!>YaVRHJ{Ewl-7Cp;c0kWm*JsK7O2x@2C5s?GNWlmg)K~*OMWIG?`AT> ze-&d&(Dl_`V(D-&z>-hPDeh;XC)!XhGgFpq*Dc!s>sPyd(BU9gLyTNoOL=RZY})kd z3(7F9LD);u|4X-M9L9uuj?3nK&6J^7)jTnc6sqKf1+RF=f$$9m>TyV}B=a@JO_$Rzx|?hqzf3C^rb~Rj zSx)8~1$0lBz?|R(a=KC9*dsYmE{o}LtFshX2D_sna7q-c!Wsh8AQ;w|OEm~cS zQM7+9gH13fSHP4JY}B_$EQ+gNirw$oaG92JPzRk>uKzt9gRhunN`)5Xn2Jt&qe{nV zK&$`V4%w0$A&ZE@G{v;vtC!^X7cW*jyMM3K)^i`+b_``U=|0A2l zx9s%z(bgDQ0mHmqjSs$>+mr3(5JbYbbn^6;Vz{DwC`Y+pN9pQFf0Nwbuf9<1k(yuxCx?JFY+N1xmqF`TUjU}OgNr{ejzCnV*p|$P*`~}(Q z$UNP$jkepfWFubQ0F(*-8U{*9?BLd6XJof&**-gEP>g$n^>p=uNPpgZyoJpr#2Tb2 zZFUA9K9!)ALp1jIF;h0UsGmpjbXmDhPDVK}c!W!dvI#NPGlVpyPqcN(40}xQr!<-> zoBgGSh7K0Lo$^qsypz=)Y@v({uun7HyBC;GG-m8=r|@1<<4UvCsV$b$C0e#JI6S8F z!o{uEwbIHZsi5o6OC-fzZ>(;&{bxtW`5)=0^OPBpvS6QR?!CxHm40^ruB+v;{387Y z(#}o^#EQnuZuO!vZ;@FnhBibw!{hy( zq7kcK3WrgSL<`r4FVcZqrmHZqcYQF$D-qlxZy}#ST}tF>MT9|GNDW^uux_s4x=wggq1IL~NGXB*qy0_F! z`<0xP8>akV@rSDBxBcx)MJByAe<*z*bl)qwaZp&{$kRy6yU4V$pNJ}EYZc03eyf>C z;T|9MFeUs;lSQMg=7CQbQ=$f8S*mDM{e+*-_P>|moF^#&UQ-sM$_uH%os5*QhZ$|9 zMx7SnESJL2n9%=R9V__bIiht=bGBO&&blnd3 zzNB(}49B=(cd|^6x=x1_jA2?9E7Px=sACaT;km@P}CLI;m;Y$861{zSS ziF+eQz0yLiObsh_JMmRlRq~Xh2F#5T0kSuML74zxH3$pqiva5ml<=4?P8R_dRh0L& z`UBqWVjbqioZX=wX281v>+rWdlnd05*0u+{5D*(U(p(m5Km$&s2meg0S0WZHemk{k zg7eEgW2y(NsZArC&&ZswkirL~XHrU6kvj~pdD{MYoorTYo_xSSlz4BKC}$}P#Y`g{ z@Y*SP>$ayC30BxyoqH_*cLkA&0)rDCpk zY%QQl)FhCcU@pD|l(AT9gG(aUEFgM>mV3ki`cs8eLr+aU6zto5m2T=_foW<~u(x+f zK>Iee`8n~|M&|ClDbcn_JnuabYY2}OJ4KuzQf>f#Z#&g@K!o4*j1p;P7O4rx#LWhn z5^MjD68Bb!OWb=6qlv)-1N9y)o@YhzB1{7s*!ViuFLf7crZ9%bxCHALIs+xv;J9at zyo+ONTd_J)(B=1Tt~KL{>S;pi!+T|Yq)#fs;_8Rk)rr=5uDT&Xm7+wG zj%7# zj>*SvcT~`s?jkk&jC?HePdXe7^o_M-#I=`<1~!TkH*hR^w?T^7uj6HdGh@_T%xF7s z!x{I!z$5xdGg$ogz;=G ztp_9*l}!hZzY_d`b*W0!->RjIj1#r`pVe&+&)N3CrU|mScgH(V8ED{RIo7FQZ*6sL zv4QOGog+^#5=?vye7vK${_{23x9~u@w~ojuw=@Yg51&1-4_A-!3QS;$# zs$*>6tWxVILK_LjrUMtoTR$`7Qv{=nERwfzDK8N6?Lc3Yye(;=0_IgI++(S2V-!=R z=#-{3lN%qbuS6h*Qvr^$178tdkt^CKODJ4cwjVl-yNIBh@$twT^eb{c%`m zq=eY@$ui>djOzZhrj+tjagam4M3B*C&<(q+d1M->j*r19assCqv6vA%h-YV7b74y1 z;ODmo)sM7JG2Gw}&mMn7I;c^qHHYgiA2Wn&r~~GsgR(9+>;f1FNg)07sI2QHN5ZZ! zjZU+2=?g2dm{xA!>20zB51%PuMr>TH^(E&|m4UDtRUX-FC3~5kVeGR9_25N&@5G&! zGRVGQ^=Bn-IymBViAN)?pkqU~<2Ewk6UJk?SK4t|L3pCI1h zK9J%8FvPZtl+6z=4NU?dOz0r|q_b2djB+KT_aA(vlRh2=4%ELu+GcLpy~S|DAYED@ zhVP8j(@3fzWk`{%*QlPhSm1vXY5;q_Kn!J!yA`BH>A_I_ zkWz!NFJ88M@uG!d>gTry-?uH;m-ib0WqR$w31LnO-DePOwJrm{m?MksyIqGAqG3u{ zwz=b8JwPi_AFV+b3*;>$t_w9&m~ub+J?|=H2c_-c8>8e692mN4MOt_RH~1*=bF<4oevO?X;Hb#OJ=tdcbN3dShSH>uDz0h6{hQ*`H4IGUOkmBvB840f9!81L641TR^oT*nThd1!m2*R8*Q6}h20 z(Oyu8=Ehyrv<%;#hVt35$@1dPrNh>!bHbPwSu6KlKdz{=Ndwrs>s(8BO?6h|B-AAOc>5#T*GUA~F z`ul<;aFEWy_wl!Sb)!nzW5}|p;xOkO6m%_i2#-NMw(B9aeqww(2d{oGacpgd59BEN;N+KO9$iDX+otYBMQ! z4$~cqanzfqrQE7}InOV@q{`dkwpVx3LmQ-sa~u`LTI%PshhD!!-o|bd^*sZ?SB*S0xmY%(8&^DF z43C$)h>LI6(Izy?i9J%b8>T=Rn+`o(VeKG^1{ZSGxQ;4s?+{z)J{41(1%au?#izPe zH;~hJAS=OuUuiE|0Oo#n0%D{A1#0md`gUE$2)bCcD4alL&}E< zVsl!5?H90M_0Ph@v&_VSZv2uqF+*;A`&|`ZreCkop$rY2kzJMzS54h$o9;XAEajbM zP@2whDA%#OFlnZQ0hWH>Hfxyog0dD{qq`XT{CJrtKhrIHx7K!Vq3rV7bD_XRIUeci z@?z6#l;cT|E*xkFscQQ^JH30Pbqn(bR5wOWf2Y7HX27}?&!HeCnpMa;WGZP0QcBTn zm8^4~x?!2_wtx>|ak|rh^QSRXfiYr#EtYNB2FLqK+unBi&1dDkcj^6XPyN_jdB?Kh zy4%uK(qBEce>dG0B29^5z4tg|xy=jcrN=D?_PlsyL9vZF}rfL z8-m(EKWE5P_SN~rtHRZcJHFeYP%23<9Z+93>%3ogD2B&?Gx84RQT(BtMTsHNvKgyi zO4~fG?N=9^EVAL?BoEk6qn)*+>8VW#6r_sF(G@P$U@g_uvDiOOi$O_`Xl)VO+aC5q zEwPFl{=txehLZr}d2hiTNq&1cPmuQ_rHm?J2;DcTkQy9uB=Je zbKH8K&HxOi#E3r8&JNt5fKd%8V^+&61oJwSQGWz4fIP#=t`cxi3Si-NWR|BVCy*+p_O* zkrIaR#}XO)V%>YZ)D&O=`jITzmS+H!E{KM}GjILwD^uq4aKuE4pBNeNUxOnfGf~{Y zbfua=x}bH@uj@W}t5PmwZL`v)%R8&xdlO?B$I={LywGldF;Xta$)4W`Ef&#!J0mh) z`qt_+fyVn8mFt{^SMJeFt2EQ5xw0o)ScH@b?Tp8>#JT+@?HO2l+oQ%FcUOG*j0T30 z{PxIZo1Nh^9?%vGQ&L7oop#i*69wH6TJLg(lZXRQiU4#{sPuekyLOcJZAQ`@*>k~e zWeWw2+qPjHdw3a^6&7^4xLMBQr9A4ZpKXuGKP8{Gmuo3VLM*v%MtOrg1MKBgXSugm zkfeBwWOd{p_%;t7rogxq%W&xo~F@ zO$>&{b0cqza`H;wRG>VYeqZ3sX9TO%*HSX}x0k)xenK%tM3+B8&v>y&4qDnodoG}m zW|SV1gD~OlP*{#)H*y%g)KwlMXGe&CykYWO8RgnvlD)|6;~5#G?Nem0A5DMyf0THs zT)fRCfl7b{wQ@5vWskgL)dm_>NA%c0-s0Nma#7hMV@jzkNeQ2iqq0sv6QA77xwaKTn2p-_@*`tPa z>G2_lIkrMTV@mJ@4_8zbQ)Bf}9gD^Gq91j@lqnpezTD1|X^(ddcU;B2V`e==MmPrH;!G#&mL11IKj91*L;z_9?cF$#J92-NK087$RJ_zf< zlj7=dof>Z=cJDFK_wBVxITH?}_SF}Cy(3me+gYWud1Jrs#^FPi3kR*%oG*+EeZ@wR z4rbnsuZEPfQ7%esAaJpTvDoMbGn@|IZBozz{d53sgAd z98vGQe!Hco)}?=(9Fd%^7#gdOsk2j#Xl<0csJe?$?C}l%p-c5wy#~|K*}J;)fPo%M zm1!gYG5ANnlk9Y)5eAWBO!#ZM9D4m{)m27w4khKzcQnE1(rTaV&)s_u7|M;zl|yPT z)gvWcV$76MXAF5ZGhm7dC^7Y{9254m4$G}dxiL8$*CnlBn|D9H%9(cfeH5KOnTvlC3%9O9|y)pFL*h(H_MtRe-@XpfSBp%e%R@ zhe#7+WBQfKyLrY-RnbVt&dhfL9BNf1@|2XZ^&_2*?42kaM(p%tCxb@}nGKVzk@PT0RN<%W(E%R6hYUoCM>+)WW;|9F^6l828)ciB zx2WQt8)^_-#c^^P+~J5PjOOs#7!Z^Sr8uG==A5x9se~_ zGDqzQp5>s9!l6ryJ$6ubEO=axKfpqj$AtB3onN(BL<2oXSeLxPW>e+;%qQzx&oQ6l zzsjx5i+Cjh zDS$F2d~8XfjCtM6_cbu}THggubt2OE<+0u4rQ}e&V#?2FKjy5Hl2d8~i?P__^`^>u z_ui~STSI?5VvoG{{joZNfbX{-&pRfPN!+^|i)~fVAK#S_e8^k43n*aYNvm1#=4oY9 zHLw9``(v_j#wLvvNmOCEl<$5B2ZZlPAN^T79Un%@H^Tb%%3IX|^Z2B5OwP>1BStK} z_P3mpM-LDBF`ao-879;?#<7Y&y<@OC_{WuZv7W1K)-9VV zi;df#DY`O|P*wOr;82OhqeIF-G0KGFUe6GxIgbk{)1izXbjVpqvZyc!l`5phH{5LP z;Y^K+Q^S<`{w0oA0^0=4c(#of^2iQ#l!7oFwoQ&Ay|_OlWqjsFtCTMg>lTVgiTr4( zav`CIVhmx{B{}NAZXL-mhVbi78AV3nDoO-Yz+^>4xN8Ql*WXHj`R$4O_quDI{!Swe zDw~e~ce(TP#Oao?oNkt~;%g<^gDTYJP3 zdWSsJ*T5z;*y7wwtIP~_!Q)9M^`C~{DqDE!UpX#zaF~=T9)Uz)S!+yoLKp*#FX1Fs zt`t*#etSZz&7zQ50iZm>hBJSfnGX9%lndSdhXI-(3#dgWZ`#)d(otxOYzG%%p zay76~MRM|udfH_ApouGUtvk3|tt^I3a6|ti;;-z!DY1rd`#ITqka6YmiRde^eB4<*pLwrplL7#f?l>!^}sEaL5RA zzUIpTL5Xf_XJzklzUO$hvKZh1cK(u3SvB~t`XOavx%k@bR~i#$aXWvlEYe~Pj2JoqDydAayuuxc85BpNi_%wrDFT~WTlwW z7^Z1^gmvxK^Q2X=J#Ha+XBbHN0sXlmxnkf(P0%Ksv^-yoWAU#x1`4VYlR9O|Xim#1 zu^RfME2-AIESM=^T1wXEP2_Ao^DmjtD^-3k5*3{6$_q)E+%L;2<;o28jnyZQI3(sm zP)v>3oA=21JIzx*F!kA!6Sj!mEQYEQ@PUv~D&{f4^Kd|z-bCcHrbKbEDe^}Lm}90d#_gu@raRu-zNc^uv>wlW|AgYn$tven`R(m8~M zz8;gykBhC1>Z$(N^vN}tP8ks{*0GYe)pRI+drIzJ(Yrc3p>0o@xmOfzzog?7K9nGz z(zX`@I@UF?sdpU+*5N{30E>Z^690@Ag{vJ6ohm+i3U3{4BWH$+WF5FuTa;QkT)Ux) z@&44PLsm9ouc|*~M-$nID@2MZKfgV-ZM0a%LKO^EDW5&{&N*T^#!=7>jyvPUGKP|> z1V5_8v=__BF6JpmmAg~LvVA8)7K2cOsaPkLKX^tl^nh8k%IX93$?Q=yekBHDxG>#fZp4xkzSjM<8V34YS=_r?xin1;QVijO%5n?%m zUaaLw8FWZ2zxj?3bd>9wAxc?hD5DB|sM2?;*g`%>RmIqYS0c7}mq^Rxlf}!WAm2u)} z{!l!UG_bWP4Av*-NHFoRRyGedwSQ^;w}~t(bSl5I#wUW1CC%& zk|?oqj}C36984kHJym?iCPokfd=%lb_@49Hl`w?>(wDo)v0o->9=gP|SHk631`~>@ zE-}6B6?sGJ7S*9WJuXAu&_<~Ls-JC7`+mIafAl>)71c9vjbrMT`f|mU zsTw*8V$;L>8Fp2 z^*pJpJjc&xXLmf*V-V-*lz{@;9^a&kQ|xJh(ge)(b2;L_<_#)ipzIb~WPLXLi7&;} zZ)f*DB5zr^OgmfKH2bw^>jy@CU{F#hk;*+!-bPn8MG9hvc93;h{o)VfsO$+4xQ(um z@}mmR|JF^@{g1L3ZeZ_q?kz7WktX=F^9p6%sYW@l!1Qpk7-d#!j6JeT$b&|&t#knk zOy4Fo7I*OKA~sV4`*^&J9JxZ5&{DE%5Gul@dWV<#2FHx)7lOC(uBbZ3cTuZ71(X@n9o0pLK^r%wCH%^cp2T&6JI9DzpIQ@yLpj>4Rr#<+N{Sa;~ox z>7X#A?PXI7pS8+el|K)}EU z9y7Ygmespn@TfuHsg{;3%mB;`NM>fRou58U4dE18cyJR`%u3NXG}2jno4M1TsH4Ag zc%Kr}3*<1?lfj|*AcduyyLexW;))1?6@j*U*H?FaGrzBpt@a$y4LPW)Fem#f_{i`~dgo`C`? zK5{so%~F=urA!f?n7G<~YX9EqFHjEmqoNrn1Ih&MtduLVo_8TV7 zg6TKm96b!iF8@jz_6Gx*2|>VL50H; zx2B8zQGG&4A$(RUbI$2xm5PWx?uruo$(iA~0Rm56jwf|1^-VAJ#DNR4A(K_)8ujPC z8*o>~WoR6R$FQmLvFO)zd(j3`Qlh-1s+6$pCmuT_A7g?CQxj6NPo0$uBL{}G_1jOr zpW{yW?Q8WZP>lWd+zuDq35QTANR*aW#@>}9=X&_b1%!%kuYRs^mho%8p|{Q=xq zNl0m{&wU`yoyZ}(43-&ml_%btBWL!TrwK9Yd;4U=eb%8XZ)qHP%5zUP4!7R-wimF; z|FGDT*IyAIGCoz3C;{|cXRTs}l1UJZ)j#PXm)RzxUr-@^a>W+0kDPA1u_;m@T&ZOR z5}Sk~*=Nsb&9y9EMd%FlGUb;!M=r@(YxEAUanzFq0onGKC-g@g(HN-vUaKjO9@Y+Z zpHD8`Cm&)p5DBJ<&Aqd}c$@rE5Ii0?#L8L37!r&do}5)Ehu>>f@r^Fa&&gqhzgS_R zg_$QKcgZM*$^(I*Yk5|_;4KgZl?i7t+IpeQQ`OF%%P9D^ z>-Oj|E+v2-wNp;>rW)E}PyVt^=6-%sn-DYl?c5ew;sp7VkziV7?#l<{n6Uw6aidG_ zeV4`YRdaOP$JstRw||^C&N&nqdL@RR6UUwZRJZKh!UXZA_g-V3!jU`oxHw)BZiQhn zYyyv0!o_i}%T@DugKXk&1~m^vQ)NGdX=(a%qSep+LjQ7xA*!tJh3tb4zmNgpUqtn^Fp0{=7#{2%71v z5u0{aF8QvT(pG`P9xSL9vH=M}>EpBKb&Zu1{u!--X%o(SWFI_wS6gh}fqgPN?kC+D zPz~ruQsh*o3Fu+`lDFoNHJgtG1XYa^^!&5#FvpTm@P82Uq7&upPf`^IgJb@)74Do~ zzv@F+nn&KssWNtGa`nJDuVt<*J`(Ha$Xak(|yY_=}+!%d6-hW7fT^?GeySE4Rtm7=5`%DMx+wa+GJWHw)yPugB>U z8v1;O3N0g^*RF&BkECPbImV}8Fi7Xe2IT!M{;LQckNa}v>LVYh|0)wwMa0Q=YiFwe zs-JD=&p6`d-|)NcVK5RU66(u#@0ux3m1<)1nRtKy$UfaCYaa8yUM)Au35ucOL5Y!5 zWxFjcs=c4zbe-E^+!0xqPvd6QJIr5FF7M}c%FK}W=TEFJ$G%`|rrPTBIZE7$G^3qy zcBMj^S5_)lY|tBJYQY>}+j+_8;afshx9#~8OXb*1z58a`-291`+?ASpHrPGj1r$5k<%t^)l(k$`0f1E1lgMB8Um)N`|SMX zMdEur7es7{F2S+JS${cF?c4l89po&gkyJ5A^S6}B##~|rwoxv>DpGVJt5-(I+4)(c z<>)GXdPp?@q{CMU#yTHPt-lPzgNpG69TVY+~9uZP(8P%R3F`|SlOJ48NN6OdqRn*VJrIi-H* z>Nb~uWvU!)X3>g|&t4EXW1YH?rW5F}Zvl_6a+!JiY+VJ$5SkUs6s`|~(lA|6w^U}f zGxL`u71|4)i4zMtn%7+}Y995k$SJH$poeK53&_$u;V}^kOP{^)e2Tl2%z_HKkrH1h zC$)G(Dds7cm@X%AtN?o&9{EQKqPYIo+_{geq!uU*jms5_0%d~L)6juL5I#2WUJ_(Ms8aNALt*x&>0Y|UfAPow2@ zuDB~6AjM)GcFHx6>!$=27R36C<*MK@Z6Hd)orE@#hSXp~#%t0l*OF)h?yC~^H2 zxg`0J`mfP;;Z0lQk~>c6;4e|1f`zx9mPyU)iy2saGm?&KYrAETJT7* z3x=kPt?V~c88`Usf{4q~V(qiUDCf5e;=^S9Uiyxe(gi~AC|RF`xk|ZA6_yKsJ0?eW z%+XfYVhiSV5L6ryQL@U!LE?;wBslJ{c_noXLv;N2ON(=owMSEnnQ|DOaaLF78%K zM^KUmR(mhX&DZS(eL#A1u3X$Mzq&;gT(?dxmal7qsWR%E9LxL`)jg)S!n9MVhHiMQ z%92UsRZt~N1T8I=gG)?qN{C3gd|B*$rk-w5Rmy~m&d0mC9DPH5&!)d!5(ij7R2Jh$ zm4C;If4ycB=I&O6MZD_vV_>0fa{zSg?1OT*1}Qtr&i>{y z9pYLRBF$%Q5sPlq`Fc>?#8^$jkc09GE^H&g#Ob07iQ@gHdfC?m|Kfp}&YOR|tS8=^ zmDoiu=gK~3E@>$m$0F|FJ@N7a%_B)``>POD=Wo+~0gG)fnteb%!H8Y~Gf2sl_C!ul zna_M)7thRe4up4;5AvBO2^UhnIW0RbGn;K%%HjnboC8Y=b;c!2l~~+qi`dA3Rk?u@ z7?}<3j33O4Y?&%7Yck!wOV0OBxCb;+I+Uj?@Ol<&4Sm6-ou)Z!cMV!JT(U zwvG>&BKCz;afriDS4%7|Ef%Y|xzw43JM=;9+i7Bz*(WqSY7%PglLe2|)?!VS-=js~ z)7iQKRjPJd_;Q?E(sq=#m`1v!`(7t_@(lq>m68q|+tTGD%z=nv73+xIoFzxrJFlAu zOnIs0WQOcdmer<|;ODcKj6N+!vORebiy5Xv62)J>y?U925_6KpWZn}G5-_I3@Ezhe z^3zp?Vru=$>C*pCh)vU8lO+TJla4v6`)C zB?ag{yRccYv#IlpP!`P;dx>*dY|rndy1s5@+P7V<$Qdl#VpGKwV9r_b8Xp|QdIWhz zW-s}$+EGg$i;>xH&A6t-U3=tHtxS?g#G>t`Q{sK@1SlLx`d@^cVmad9X>yoWWU1>U%u?=*q$y z8Cv&`2$ityr&^tMpJXd-w+RPuaJ*pvvql9yQ#GK%hmMO~EbOWh_y(~~K+eEGXlkSs zmP9!7$??CLb~{CdW#O=O;@$2h=^dr8>Benx$y?^_YwQY-XD`Slds3AIyBnVgeY9-5@Mx)+!>u&01i%g#i+8w>d=X#}{r1w^GTpgtFY1871U~zz zmfOS^y+`XMHQ@X0r>5@{Sxs-!Mn*I|SO6&K8mrozk_5rsC=0rq*E%)fsf8)>=7$bw zZ8h|#HWi6C+4rEnr~a%I`AqZ`ov^M-F@5UwEi#G?h%(dDL1=eME~#~OpAf5u=gXo8 z2kH_a_1l3}tKFv;y`pCYL1n_HZm2Jx9`=m-?kjL8_d~8MjvcLsP8!G37UlA$?q(BI z`(pHK_lPyRFd!ED*z-JTx{J2)k*yH(hcXs>R^oK7`*<YRv`BI(S@$B(3z{okQblIv6ba#|2FzFmT z_1kxpi+VO!@udh@{f|hOb_kb?I_ce66NO8E%@MCH{#7>*t^Ts+8{PcDGu38{>f8PH zvcPJ0{$F0wA%*DP_R{~7d$2k&tIrP9+9_M`(aHoE%GvhP9sA^@r{`7&ZlDoqz3(v- zJOh2mM!BqRhJ~h{E;R^MnQ}Pq;h>8$5(auWTHcb873wY!JMgG%dF@kR0d5Os(OrPS zd|m_FE{HOYiDL^@&1Wy0U&|evU}>uxrf#CVu6dj4uW2B2ubj5QWPi~}1IPEuhgmPY zNSmVyV!w-&4?kF6g{2x#`RwJ*I=D~1I7%}mw2|`tPIpkxqx!RkSWI}~iaU@YoWg;T zAf!Z#PZ{J@8ekAw%#q#yoS*~7lo(Vj>s0tduJqf>Um5Q%>@-s&ML`H(opuXZ4pTfd zVcBbY|UofKICE%JO$}-G)cs(+d%sm9|Aw&$@-!qY7sSwfc3B$^PX1DglR9e|f#j z5PEe(toqq@k#CPXixcju3Z?vZ(PKN^{O>R5_z0-pvS{*VH~;hLA@w25zUo4N8v7D@3)zNnRu!#OI73se;P?G;_a+>u>R=y@fj&~|txIc9c+LYoq! z^4&h0^&*02F%nGA+qYU-+<30227$fC#CN7^SC%NGD^^Z*TX2|zKa^d4_KKnkx5d{j zR3Bz|tSF0hr-dEURbWh&kN3G#|9zt-pgx38YPr+ypRbXIsS+z+Z12t^p&|b@A+_@P zCT{LKZPg#94_Z02i93Rk<3+U9JXYqOawoH5gjh@SSW%KHJ8`@Lg8JF^$|GCc1suJ{ ze9H1Zd*#RB?t-qd)r6uicFCtdN(;pYzzWvMsXGd^a(Noqv3z%E`Eb3or`@n(bvHTh zE0gAn620w}&5LB5_h#%?6);ahUN!`QA>JTMSd0_EZuw_7Y)B|n&cvd+1iH^~*DgMz+_C;YzR zs28AAXF!R#LOHwe4&ARC9`_14dr6_{7}d|Vi~oL3?A?8J&A?}`s(U7QjO{0kqeR2B zd3$GK)o)@Z>DNZ97FGPqI+?*VLHSl&toYpsndhC^M6_+M8pC&v?6+k|;Rlc0d^wp< z0mV&~XZq(ly?NIQPzs#iekL{7N#rbkC4``I1cPPU^Bp6q&>00iJf+&46MN3(Idy!n2{hp3> z&0~!(;0oT4ia|L|Slc|)T~>I9&SDkRNVgWbop|yR3C8&?XHMkxr{yWFnLPU^2mK=nauv$JFY?Y0uJ5GXP9khAK%d3(VSI7l8#))mWi zY$SP$Sa)wh@Oo0oCBamO)Vip2ahLa~9egP9aHR7PbJM)gKwy3RYPXodL|v+& zF(v-C!|g|&O~tfIL(i?_{Ef{e{!rBr`&pLTuXwN;y{b}F2^0pore=kOps$bY=%zh% z&0fL!m6zSZdDp4&x{^edvR&2*a0ICWbeR72vh@XrCo_>^pr4EnKFfkTpp-t87PvDH>KmJ`FQmI+2CR5&Robq9a*Igdc=kmI{2-J}weD_XUbP3U-`;Th zUbpd;xAbx;C4Bb!uPVBK!P*v@n$B@Oxo&=WXP*pU4@>}$(}C`9bB~V_%HdS`F0%U; zvdwp9*i%<)d-hECb(E;21UOWQbKQSsR-l+_vFk5HcmMd>bJ80k5}X^IiQ32kWx@^h z_gcSld4oR;jtvF1T>H*o^g}8B2op@5=B`5=XK5lTu{b*suYCHrOpG~t_7u0ZexQKXVqeJvmPEkdm~>D z$qoPACLFNG?~&pjE`U%){g9HI=??EPNlEYsH?I>{emo~xrBREBK;KvH9A@uTmB`ao zI7ff^%j4>JmHNTs^=8gVhNFm7Kg5aA%#CLYD+n+o!DDriSngc`(AsXgF2aq^KBJ9{6u-Tx-BCB?p$CBARoT?2 zmg6ImFTylWh190O@oopw@+x3Df;RqiR-B9bNzbAY4Uakd+^*ybucQP7s&qIl9wwQG zvU-^Uw(0RL&S7q}gF*c$G5(x$xJvgw#$ubY0?y${=QJ==qVECcz_r1@>AQ_?UfrMc z9}PNK zSF!7ho}e3{(dC;a&hxaFt8RF9Vn=sLv9`Ez<)+=qVj7#m)K|J7<@FRXjdXOFPjyNV zK8ttW;?w965L?;Bd5hiwo2o|Wv!5Lk=5{@`LZ3p?HhuQ-FtAE2jT7bP5R``}1AFDQFlfH&Ej3Y*%oE22dunpY4<((pX`EU^?k%B{@WH zd{tW=a8wy~+@sLKnvl+b>idV zZ}q1Dg3sREn)5-tA_7At$+kDghlyu6{SyQS`uOe5&vtNQEBv|yIH>Y{u{-6~hU!wE z3M)$ND|BPG>_Vb9cG&!5H#eU4^8Xln*SM;R?|&Rc#Vd+p;uS+fK~#)P#qjKz%~2G* zu|*?MQHs=}BrP>k5k*Chn%9IvQMsrnN@Q7H&fdq;C@eGzv9$1$W~HH+rk3`9ubF)~ zyuSS&{2%NGXPud~-s`ew&t=Vw3CybQeEhIO!v$<}Q6+@b&-pG*{}Bh10D~#zQiV?E zTS9_KIE{>M%qfvgC62S9jAKC;^`*sjy8GDAVQ&*)%fQ_^I@u9~TXWzO*^KX8=;i2x zVG~fwSl_7~a|FF7hmiuu&d~*qAnfFda=LP7-%G|IER&L05H)otZio67NKLV-x-;44 z=(Ipw(?q49oaLC(Y4z8F2TFmkvCt80#SsQhd+cQH>{}#XUj|sZ0$+H5fWV)il5!k!4C;_Hcb=T%W zj)334a=i_^n`St+4z>%X{sev3Yl+MgiA$s{9#!2nA9;=#0+u;_yDKHmD8Xz`3A`_v zE~=UXZ;GKk=jR^b?>K2BqOldvb(@mf`u z=r|(+P-5>%$7^%iiW2$4Q9sV+%ZFSRpXi_HMc%zS($V7UJmDx|)7^7JjA--N)l@R_ z?wNMh18aPQ1Cl^ElHq8vw1;rRRkHf-6*a6F3n67F0gX{2Cxb;`m_XXP5s%!ao{p&_ zUJ(0$mH#&Zoh#PpJ6b%T7SpQc2G%$%E9K$x?Z_{???)&!6a^^L9{>6%;pSzOr~)ZX zf)x!rmZ7N(huj%{j!`?S#PpA_SgvKh@f(f|Az-jPszg^9*Kn%nZD)g$`x0}sU@wc# zCX_>o^iap}#}?A80+5p~<1Z=C;KNr19zxsP?O_Hbm5_mCiM0hRw&4hpU=sc;cJv*j z3pW71rslQ7#oimd#HvP1a%Jvs6~@0vumcY1%3NE9r3l075D+BO8`q8HP#>DgMk&uL z+0hp%yd(ugtD4s)$@mPr^avPC04eGf#~7RsLrj4bC1US8hJ6v{e8oWM=b;}!!va?N;j=cKcD0j8|L_!qAjRgq#=%@i~!F*?D~IzVQa3FY##{2Zx1a-5A)C^7Pqk&7ejfk6gc-XMSb z0}LcmL69Yert2-Sm;wy)CCrN)YrKQQJmxx0m=_aaLNH8#Sm~F%@5efphR+hg-%+T4 z0sWwzRl@35gyf4HQ`k=}W(Z)R}=8|$T#<-VMCYTE4^1Ti_Uii#cwEU9CEicCl-;Z^M19%)R zXDhInl3M4vBhhRH&i+AtIpEE26=YnSWf3M6ObV6hs(>MVbl{gSbBqS~G`e_H z2eBR)YGfklB#z#gr$bvf9Lr$BFZn$^jZ2sgYYANt>zbP1Hqqi2e6H;jY0#%0l8OYfb!Ed7|%+IwumSO{06~v+x2y;@5k1#0^f8OQoF>Bop2C5>a1)gDS z*=P5uz64hA^bunrj`Rm88Tc)WAP_zVZ=z#dczgjbLlTYo97V2&I6T~3C|cfa79 z5aYm4AG#b>@N>Aa04b~aPMiL>#5eQxWf(=1SipyrOAn39a>|vA1j6Nq#xJCC+_Ib4sB!M$eQ6%9%mP3sq&4&h$71)!y~g(>)|khiJS zoc&~2{ona|8n#lye3I#&7V{lV;0Dx}NcRjZFz#dgVgo@S-Sc#saX+h%lVDZ%-~zU2 z%xxjQOknq8cmu4P5>u(%qe9*Qd*4Ew-`hQHF0kcT^Q00{V&sm;?d2U%H=AU(Gy z@ra`(a%nT2;jm}S5w;XBVH4nh6o`EWCz7Lc)g8K|3c`kDwgwCSBo;P7iA_GnDMMa> zA_xJ&H*456(}d6%RkFN|;_bbtkvE~QtknnO)W~)S2U%s`5l1W60u`#P z!$||U><1D|9P5kN3god}1=1=Cm_2I`v)X6-3Fs&_Qr+7q!1%KNV8sp4AqgdL9Bzj( z^|spYx$w|Pz;Vg>;9(QG+{$uv2(c^e;*D1Vv z$9oxLZnhPEfuT`l)I;M5B$Pp<)ON3yW0cgNQQWI;Qr^czCRCRZv^P3i`2+(A9aW-8 zxxL8|Mme@35DxGGVL^&fo_J4WGf2Xm3%mu^=ZFU$lvs41?ZORYHV}XVgp?xgar!Uk z8wFsC!?*{MIALJ&3kXYmK#&v6sD>&_FL4i?xKskMlCU9&d-NYHctnX3MGv_L66;|c z84i0_S8xwpLQRCSIv!v8h|ikjv96l!zb{u4B;n|6hxhec;sruAh?8Q!uS|R$$ z0@z+`L}`YNY;b`YL1-KgPpG$Ve(&k#ywxL_rj;*%RRr*+BNOT|z0LG=+*G62g+pa6 z_Y7D`z)Y*xRVYWPBwL; zklNQTR$qs%w;NO+IeB1GErhSPVFT|S`rwJ3TuWu#}a3J>?5R8fgXj047W5TUlRxUh%JNKlc5Hfc(?G|63=JMhR`tda*tG<0}W9)(fGj&@1Q zL`cCVDDh+s8!S(omq-i4eAy68W0Nj60#?Yp*ifXdTyFn6OSH=|K{h?g+7LL?fLF9AFH>u2{=Pe1KghEWPgNii|rQCBPKA zobcAqt>D7yvI=gdbz3e^D3kVh6y{ibTHZ4PEMWR^7TbyGV-z=W{1olzhHF*xVdOHQ z+`08^r#V{;9*}Zxt0NG{Yt2Q&CTzOD$1z7AOsT~KhlF08#k7q5POGEJ^}9^_d#*Fa zAolxBY!_yWVLsWQ6#lT4?ZS~%s4v@(!n?uxK`iM(G3mMep}~$c9IytY64?HB{toZ1 zTSzQ*(KU5{`}>Y|9W;UCs__0TFZKr-CE_9MvA?UYqdoSmK?=1HsM2J;l7ea5RB1vG z0?Hjd5k)?jUvEGL+TT0c=r6;`3OGurq2K>RC~FE`U<(szV4{6Rq>n&~ z65!FllC}I$-cLY+4gsE zC?fPA1EEBiJ3EW9DSnt@!|t=QPwpb+=%Et-AzY^tf``OmO<>QX?*MDj17Ow&_AAE7 zsjvqmfiOOXw|-jQDr*zaAM@6)18F|0NFJ{Qvjym};a~YS>|dU1L}4-ms>^WLzr5HO zjm6k1AeBsCs^wi?$tT#P9FV>`n|F~3hcc4)FTKu|m{<8gLExC@$-8WcafQ_W%w%H> zCJj|Sex;85aNFf>~v1wZw4s&7>fbje`p=IpsF@#7}899P5= z4oCshYvHUF=08ZeR5Ia#Ub7t$*myw%lmg+Yt%{06L`{pK%ALgNIMgV}Q`0Up57B$(ykBaRX>k`MSNI)a|BxA*=)`@_nJji=~e8T<#j`6)f` zoOJ2}9Nog073bvur93hAK(7#`2e$0yqdi46g8qLA-LENr81EBwS>pfda$v||Wd^c; z;5F%}12eK6UP$-23IkPH=h+~2qsY4P7dBlMqMtw&=pw`6zz%=If#cSrfCKRX9^12-0;>}afP-=$yF2kO(P2`cpi64jmHwFg%RweUl?P#LEEembzVu5`i#Rq5Ggl~1k5w)5^k9>bEJy^1 zCguFueJoGvz=3Kg(QB-6`C^o7y0oZO7H@$sA=v{8qDsf>Y!tTcKy_)tqRu&NCytdM zrr?1Rz2bDz6rLj-VCr9?e1UEt@sK5OtQ8i@oE;KW=@M-mMFvqmsznLqo~*E6#N0S| zm`I;YW7DvPgZk1QMT5_X5-4tJi-QH?%`KUz2C)N^S^SKLF3%M`9mY<>gd_#Jpv0&` z)&qBQlt~^SumaW^%HdxV$M7(H0FoDo;3kxdEqC;LL8T>5G6A4RR_gsQS|jD)3L+fS z8zKV$KTKd_{PZAsRwofhszt*qSgd)of-aa2f5;{xZ|im)kLY>I`ny*}z(}!Ah4VLB z)QT)Xk^q6R1CDZZ%w#6)gQV^R}!8Tz=0I*G_>pYBN?A6EzQ=0(xA1Q_Y%3-wpZguU) z{?7V4*yT%Aq?EnB%I_asbD9uR@?+VS_v+n%b49V=Eajw$7$ucP;$*IcvK zMW5#?=g^+s1`b)}%VYXlbb6_ZG|-7yHW0fwQ{kwNLdxmctS8?6k~;5~cV^wN#0ix2 zAlCB}^^4{ViG=)nqi~;?JV!am2aoxc*RWPmcW$mz_ z>jG8@zo6R#P$sZT-oxHBx7wo$cpNQKZaCIErGW4st6yGk;J$g+;jTs30$5`l2%^`; z`b{_HgU~i(3`p+=Gr!w!31F}qnBJbx8kl#bg$Tl5>8u&TvJARJ2{$}+XH9VQBJ63t zK>wG=!A5r0u>MHA=zb{ggGC*pSYx~% zz#zSU@Tm;P0!$85H~@=PJ=nUIHNoeEbP>Ki*wT|-z)^#+mW+>sZ920I3`J2~y5Zns zN0|G9cPU^@vE!21FpNE_oL{knU1QkS!6I9Zm_kY*w*ARti=_Godl$2(;{ph!RR3VV zFy@ZlIPnk`JE){kMuZp?0WEZyUf@_cCf^nO2l1WrM8M%+6S|KZpLgC)8v~ge#mB+% z1;%d{fBIL3!@;k89J|jg5+5AlsDtCP*g%}c*bzMNKne~J9*EZi^-Tk*;^-3A8}_(L zAB%(k(Kar{as>TT;(=-?@mw{tAi1T)`6B<9urR{-9sB2WFa>;6Syp6(;RSLbU7XEU z;3G~vQiU#WU1Oavkx6a<0aaEXVY6`vPd=2BB?__Y2n`!mfn8BzZ8@8U#HA1r+yoD- znonCOPXvt;B|h+Uyl4|cNYmgbvUBa?Fvo zM&3>-4fJgg`#g7+Xr&~uU#=LBVbY$I!vk$?(Z?@gj|4%Nz^t|_u@CDlP9J;}##$nA8ud*h|2vkoT)|z<#UXbe))EW4q+A)jK!>_tbG%~C zNyB>}xHEQ48R>G=A_a|hlz6xjKV;w@YI}+|!0T}vy5IqiHfMN) zLpR8L^r-3~T%5QXdE6b%(9?A^NZNWHFTM*bF-yd8b;6vlR!t(Lmhqa_%4voj0 zB}T6#MVRo=6n7<|e(KvH9Nb$NxsQrVM;+Q@cRU-PB>FI@2IxB@9Ahw9-UK)R41~45 zY>{~s1FC_r?V4VKTe{t(*pcd?Jexfatpg#IRrYMv`{GUv6esmjqM%mqhs>RhuqjG` z$0w(hckva=cfPlWzQ5u~`{HY=VwTX)u+zxvONC7()2|ZsA&}tWQQlb(!|oR1fl^>v zenx-#NVMPqJ|NV@IpSLm6}m{V=kK!AcgKoe4&bB2PbtcKuI~%{f~&JKv1S^h<5#IfanD@GW4KrXKj@tS!6RLvd)~>nlF(w4(i~I2 zL9iIK9jO+#DIT_M{B_p^K=B}EB%=$5;)De}cs4s@JaV-`#Vo=22y?KBkP;%Kuo&Z` zbBib>!@v|(#^$i|xCakE$k0|0jhAp@rPpS!F)s}F?(tp;f7`%U9n5WZq$ zQ#>HFV65A%fnv}sw0+-qlcNvrp-1`G! z*af_V042q~KP1MW{%IL_$PyC*l)2`*X_Tlk&0qQ6>=&kzw(n01VZYnZTrW6@}Vn5Ja=k)RW+rpLfw;GF{ zfz&oT!8kNvvCu_I$@VptYdKCgK+5(=5Lyc8O&xBaR+842VjA={U6{2-9+*dbum|SZ=|Z`Z#szE;raK@-27gH_cXqX(x3Gu6 zQ4(>+h(&f7j3fP0;$O%vBgV);vP58_@;73gc;KS7oon?w&WSR@-Oc6Ybo0gl&~fY`!rnc2LgN$h9?d)FkeU(0}n_U5jPB5 zBAhJ@2-bpOrR9SNrHS-K#)>efgyQt5YRPPG)wjUW0sUQe9lwf~^ zC-x2J<>N;Qq@WrIw~w)x5mN0yC_^JXY^j%*avB3(8|*>q58Jn| zkd4R9fy6@)j@&uP{>I*WXe*cw)1S^}e`CKXRUxEdm;c#nY&|yGX)(=$>tc*P*xE%% zNvx%LlM3Y-EQ7h3pJ6sl$|k_A(xg+=texYC51W`;M+jz3m&_1 zg4IQI;Q(Fo*o}L@^UWnaQKbkQrV0=rb=rP4&(oL)SE4>KjkbK9?rTgmzxyE}A<>rK zB7JznS4#<))b{HqYL$*Fya{{I zT}gU?JWpDxzYD7~cd;X+#Ppo<ZBDMliol*XEVn60DHmHw|QEeQgg> zq>s_?UNe_2{jrndy15JnJ{lW5J;LyTa;8nKY)H1dF$v=ah!hPQW1CZyG|a*Qy3}^m zhuJIuX%>(|kClx%dtLbpqXO`dV#m0-8KV(n#1!B(HWt2(vd$FRW)fj?s0RzfLaHfN zV{yIj8qZ-Hl4(;bdjTh=%-htOb%$?qNJ>1c9s|EXq{Q(&+OeCM{sBX(rn1?8)Uaoe zssW2ju}Rn}H#hfz;GqzYq?{yn9wR;*VjZHbZ0^BQKI{_@N}`aCdad_ecDLR@g1x@+ z6uXBrnTRQ*Xlx!|9~Ql+H)$%G3$_$_#oJ^?H-XdGvWxCY+O^J3ZLMs@3_sQvy{gS``gbW0RkeWa;pa{Hd1kLFZ7U1Z4Z+z=aKybh7!lR zsw~Snxo#?T*;pHEEJgAYl@Lrfuc)wun{R{Ac1sH%%P7~ht;V+bZc;K{?x5gbdJ#32 z*>Ju7uX!SdfS!%J=6k^VC?QRVtR3!-WaBy<+uNQpRxRpX_l;uPk>xtF?=oRiVW6E| z0@(FGrZ@veV>xXGv7}OY9*QjSO^z`VTQb3v9xKbecGnn-WDWv`M`L+k*6N#)%waQI zZMFwV2i^X+P6U5PiqkzmC9!TYZCx6>p`HyfaohYQK-24Z?k|_*e4SUzNo& zGdj7{->>RZ{tiWI#e z)UV@lC@Iu_2MNn{zz%&Jq;xlTlnIL+dbQL@LW_$Z5_<86*-97l)*TTa@4pvdOu+V1 zQ;Nn)-f3=sh_5=~zyreO6#d88%c7Bnqcry6m$k-7tO8OA7)WCuH4igJ;go<9curLL z5L=lRqW@9H^b;>HdoWI&H9e=Xk3SAFlF(1i2aiO2#b}`#YH(uqH_gFBU#ync}5!k#Sa+{XMRstR@BR< zW97}m*!bWZ!ckEezp$!K^VrC9i^Mwz_!>L68Y^dL40NXn&)th=z0vZ6AUapk!^GHm%(#n2V5X~<3c}8Ua2zy+@>v1qjjF7~X3H#x< zLZc~;2S$BqkBejDj8SklAr;~9$+pexvdgnq8D zn5gp0FU*pQgduVRJg2d1841c9=#dWscz}R|O!uvi6IIHH>93z>=nV$GNj*ntvELf| z9{(`g-Np1qMG9|QB0gy{q;Age<38}JFD?3nol5;$(Q%VEc@_nlT(Sq>;4v{Ib@53d^;q zFRR?FykbAK;9X%LA@+7-oTX--ydc!SjWhngLAE}ANSofq0Xh}Pw(J@IylX>#eQqC9J%IAeH-TPiOWL=GaJV2|4OsK)LcHN~AhV{Da{4Hv9ZQtFeD@ z`$s!$5ziBU>rM zFo6t&N8@e}JoIXev&kNU&|rbx=!9)I?EpOl+H!Y4Kjn=TToA5OmBzasvgKIgO9T&5 zqKVgawhS%IZB!xUcoUy!B@gQ3U!fdt(yK_XLZbwq2v>5C-)nU@j0!T%_k}m@8)^*0 z@+6Q7tMg`PpmXJfyRZjup69FofS@C*!zNbV;>lA+6nX~0A(?s>d9%x&L!7a$anG`N z7K)o#qnu1V8-*Krf6W(tDS#Wv+vpdJk(kDIb6TwJf@sTT*TCD#+d2GLHHJK7AfcSM zXBw+Uj~><{##Y|8c(E}P-+fZ94DT)XE=@4{j!B~*1kB3Y-yCGwTCabK>!!RcyU_b2 z9>8Jc9S#&IA+7`}jd#eoaDrwPps7^f2a8jA=7q^%I*$AN;eDbpR-32<2paE*F3Sdl z19YLs$~(v7=5uUw#lO-GUFJ9Eo#ER}m=0S(uIzfPv*L|qbU>G`?6&Hz{sS6msuD%I z_|8gD`pHC4sa!V<-k+E6z7*SiVV2Sj{SOqEp6mXEuQ3#zJmQf`;qW-d9_x@u4o68G zk2g!uFJcUI8}U&>G1mPTX2f94lS;rIs1jOjyo&B(-C7fL?Lj1a4f(`K=-Q(Hq|LU`aeuI6wIO2`6akhwK2oI{zfnz z(6>MtfZ#`Q>D#An6z~Cw2gzE}Vo&{%#s|!Ji6qdY@~0knpu{~F;mM*JBSpEOz{rwE zNacKG06IowAc)rZz@>+I2OMD!r1V(%z^6Rf(*5yHduV*{_9HxC?puOTM!JL!w)0-* zggeAq`Jl@AC&q(moq<9F!+5}3Tg84;paqXk^9%*uVn~qo7<@O|U}$AfU)pr&npz%! z5l*TM2aSh)n9X}!mHi-;()iG1oP&lrr4o2w7--m_=DbH6nQJWxHF3Nrc)$h3)XFWb zaMKlVG=a7-qxS!YFvi=m2YH_mNm8P@MGLX)!6tb8kkBp73ytx1f2TdHoU5~Wz@EEK zzi6C)iJRbmcHI%el?&C%Rt&4JI&mmP-o{XK&Hx6oa<%b&-eINdz+A4jF6AAP2MQd} z$I6-81!E}o6T+1wN#jhvaN-R#235}HB3#A$^6?4r?u!!Po#yc#gPVyM6V49*Xg+@; zYPr+d8V^5>gGd{Rja#yH7~Z3j_d;U;H;^PNkNB%O@3kpM@PJLMe0W8Q{T~>abg_|N zM)WFDmZAwJ0|~JsahiCq53f@FFt|bEg7C>Wm*<9Gk2fMQTmdNQ%7{;@jae9sfFOB{ z{%0%ifrA#nP)dnfbjf%X*~cB7jnbHge#%RjAR-DOlWKywjTKoC@xD(c>M6N z4=^bUU=rzs@m|A_Brn9Gl*T8Z?I^~S34X|TZhFLBo;f^Gq}7Z?Hnl8x|0c zxNecgCisPTM3J^r7G2V}V4|uH`V{{PVgqy013gDFvVcA#CWhIOLjupy z6V2l>(s~B7fmA{;P1v!?XyMAFw(?mGBG^|r7{z%YJD>HLjqUi-$H_zE^FOokH9J0{ zz2>y_M&omjjumGj5L2P;T+H_DKmr7QNF%?sFo<2mUgCV15Ra8FL}o}iS~y4%=LhlB z=jItI2F+FAVWY5Iij6(I^&hULZ|OR3CCc{)@tOiUN^Fl*CgQUODbj>X&u1tdJijD2 zQZ*}2n>62e8U51|*}_=)vdKQiNHb1_`peLaKZ7nDj6;u=FL$pr9>aVgRRIo-uWWi( z&p|`qQ8EQ#vXA~QGVQ5^F!CzveU^?R?Z8lKyK+T^F%~JB_*Vj3nH_2j$5@PblnJK0 zLyQq}6N%)p@@Tpdj&xJAgq5$nzE%GPtANl&I%?(l&c+x(NAyS@>z>)l-x;x5OhUq@ z8ee@k*zm#XN;?1pIE}AKImWt;C=wQfSSw$XU!gz18&xM|jpL#IKXV2i>Z1y-m+yf2 z(>j~lE=B5>kq?9Vghk`Fno|AOqsN4iK~TBvcBTHSYYqj3fBp4e%~f-tweq!1JL^~R zd9^tWwE3Vve{XsZB1j{zAJkll$3TZ|G7a-V&_x2v zc^IlpHK&lkN8>rz!We_WI~dAlE9bx^V}VS5lIrig7j3M@K~E(RE2`wS+{&B1--eEE zWZg+@ep^5NA@X4~^k9dOP4j^x!(4qs6_hBdv3HFcsz3@oR=$V(o!Dmgcb=&d&G-6O z=+#j}SumL5iRSydUa}v>(oYVmpbiMH$Jq4?^C%IFvi?Efu|_DeTp@uTE8myurgwFp zElw!wPEra%@NTr0ic1~`zushwlufWy?og_mu^!jnp*~q&0@Gm_o zubk@5Bff4&|H_c6M4U!w^}eQ+ORs~+I8h~vtX`Rs#UpT9G=7*&(f^2m)nQ=rt;)ZP zeyFd;t|M5B9xJbY;y&MT;t1`2kPXU>M`?WD_<`aDLJ5uka;AU}-`|1Mr^m{F`Q3*P zM^6dBh=<01ZM}(AVB!uxB+_3u<*?i47DP8u0>kP`9Gi$Avcz@IBkbsi^2e|8?*87M3vR^vB{BiOenQ6&we@tXAe%#g=@$o8Y==WFZ>%sZe8sju;y z4gJ`+NGm{n$@FGm82c9M9+`E7zV567-3`+~V7fle-UA}*2)FiCv#;>cBIttYtvfNS z0!wK`fT1;BiyOz!PHHZ^pGw-^{-m0fVk#VBWk}r)skAId@U;9xMN6dAPj;PQz*Gd>bD1-^4yg6gpk0y15@? zI@&_wLC{sV7CCxza{&`bHMP+@jBmUy@1lS{n(FaqI4^X&AiBkZ;AV|6thj^;^@%Zf zRF>-Qt|=rCsw3>Tk(uEP2i5Ivu)U_ZFPsA}Vv zuknq4%VTJTSoeeeiXVa>stez$jq`l@^Oz!l=Y@gP#*wwGE#{M{gaFoLZGn-w@-I;W z27)fny4h(G5>4F+SD77QiRwhaqe zf6T*>a%JS(wn{Yn9lLBuEFP=cw%-xf5%W?MFhXqG1$R$m%y?MWT&QgqS12RRo}i?r^l+cuW8Q4V&b24K?xA<*mUPMHGtF9$3C;^ z9g*lqB_v_xd_~23Tlac&=$N1%M#q`r17_6J4)d;A{4kY8CWL6K+Trz+mdDK@AQVK2 zi^(h;;XoxsoOXDWt5N}NbLjt|)O{z0r%zsu)?STOPnp$9!ywaZ6LpMrNEKPVic)h?BBYyehQY)*@HdvZNngzcrN z5Q|4syFD4p7GWNjs^C%8ZvBoJqp;{_df%#c3ywA-T}c!mydJ~GU~QgwpbB^-iqLP510}n~?#ks|eaWZL;GaS0_Niyun zy=dq3t#6pS{u9QPU`W-hs&9<1J`p1vh^0qUeG{{l5~QucYclA3`8?x?6MKgS}iGT+sjwc%vKlIUgi=^gxcH?dM{GO;4cObD@|19)b=SB=nveBA7c~4-=_X?YZ1j z{~Yr$C6EFmf$(0ur5H#dlB$64VTI+J`Wte4ev1=|u$z=hfR9yuqE{*NM`HkyG8~== zPGY^$fzL+?ltKyLE5;P$(h{&T()Nk5rK}fv(&Pq!v#S32H7q$P)M+tQ_21{g=3sjm zB*-fMXS0-2EIyJhAZn`rrEKLpgf?IkWALaxY>B|7b|e<1G_{xeY-0q{tVjwr@5rw@PsldI!d`Hpne*0v_O_slDHdXW{r*=#bKpnD$r2_r!e@ll1=GOkO#3bZRc^z^L5kdni`0dfG*}oE0drH zUb|pSkuRY{+Nb>`qcO5O2$(3*r>KU%jNSz>kfd>HpM_q`SH?MMtEqiTD)k&7ty3#LurXAgG9s-W6u6#wxz;N+HK=E?y?n_lLKgu&NUdaEkdo?r_k)Jl zu-33TKuKUhxQaB_++inh1nIeKH+FLXsRS1ENtm83_nONpeV_5<5onYkfh4JF-_glN zj2u`=V0~ZR#0qhvG#F9|P3^m+IS)sCl*o6^s`gz!pN9v@;|m}Oglu2-b;A)Z=zWW` zxcW_N;z18ed=SsWv4UrlR-&kn2Vh9PV0Y!hWIij3$_r1qOny zs{Kx;pFm=ptF`R+VXzT^<^0?7H3g4*gIFspgP}Ny9jW$jcZ{_`8-$>hy7Ygil(obT z2oekEn%aN+5u-8uLM2cQgraQL8oNoU1h9Z`JjQ4&7ayq#m|`T}+B-#jUl1cG@nf9* zTVzt#Y1{v125-FaDPkyX8r&_L1>q$mU7|?a;DMLyWf<^6ti%zlrSry78oc<3(Qt93u&ETg+|y`; zBUDnQZ>?(Z@&Ka|Hqw{LaM08NtqPcihK-ypstjl!!y++!t5a?O_IXAkM*y}UqE#Ib z8mn)S8AQZqq&lEFft{$A7Of80bC*w>X@{vj=%T41SmX-)xw}|h5mG`vy21l7dyXGcY)H8q z4-8-H;|0pQc*U;5`y$2$ zHk`+!<;L`7tT>>2)>6jVr1Gl;mGceHuw=qu^Uap;^N5kTF;cOr>w1t}SF7 z-9QwVNC)}FGKSk2yW`CU7EK+*=jj^`O?F0*RULHZK8wa^jkFaw29;l6cC1AwLR(?v z!MI`fQ#7zq&R1;khuQjhnFKDy4#t6XW6ZBQs%h%rlRm5!E`TMAp&AHPLF_W1lP*9D z9GFyzZ&52+ONd5^Un5uz#vUYA@^}=j6EaE<s=)dIgU>-|0TPXqknI7rvjRNL!_A{@C>c5)@q@apbeFkIG*Wql!0i>Gx%&Vn* ze53D#2_aTf!+vSbmcx}%vMn5^4jXqt&#AW#qz;?x!zN>S0re%YVXN*MtFb2$K9R}| zTN`Su3Ed_LAX?R784>ImbL*c_ZdgVF8;LfVa6my#wLIx(^gvRp4Fre=;ps?yEm|r3 zpb}Qq619~b!3l@Z)GYCQywO08U*zjy+18m2L(V3OOE*|BZQE5IZy}@5ax8^)#mRB7 zg~{|_9J_%@CNPz{SXQ56cd(1I#HovAO)a}~x=73@N?_ktvOlnfLrC!e>CFswbH^yB z)iqT~^3^wD{sWpy<&>9>=^JIxnS{;pdLe)ThlH+t8_Mos2R8x310D|wS(;iRv=s(o zZE}r<7y$!=tirnb8Vm65*I8XTyO>Y>JVTTyBVepw1*4V@C8Ryr;u`i1zUp8o)#ttK zdL_nlnbMV3mE((i4t?!96Y@b<*f&?|u4M+(`-k~r94Cc1l@{aE7qbd;eIMX7m9K+X zbXiO(AiV9ysvC+EbtH7Y{|pO2nkR}&ZFy}hD?$rHZU7cdRWVt9&A*!nOJNT+p_2XD zV!E)pBz!a5_%pDOB1pDl;bAe1u}4A|!6STP8asj}*aj)7wC5~*Mi`%n0UlWlk5vsf zvREnxymdIje~qvw<8uP0q@bpT-?7=JA}y&dKCC@)=d^r9O|ifD^Eu|hMkr-ftwHfz z!$egb59{lp?9=TcA4Y0xEr`?C<0>Bl23=5Pe>}^L+(#&-O^3gp%vNE|0t^YMrVjro zj%8pbuLRIh4TR(FECZ>pR6>Xye)1Um6icm9U<%8l#OH3j8PcQb2qW4gv7G4<&N#KI zBSyO$tDEf?Mg|W}9T9Pt<>4H7)R#C$U^3$qd~k>$Y&rthL+4>W85m4Lj62KgC_Y#O zA3UZevRurtqKd>ZBEB=r{Y%7HSQ$Yd5pQR?vbm5Z9I-e}v0=nlh2RHTl-N|Md?8cL zOeq^8ly%j0&{cJWo?%?Xs}4}6*a$bCh&eVgkmQkjO<9jv#}8@35x+e&zJH}gl#sgo z4kxJmPsu2%EbsP4J zfgwFsb>!0(Y%&f)M{()fk$c=&*zf7i5~@0?vyYLAy@0T|RBqIg(qZpms9{>ns*bvO zLGKI2beK@sbX4tny_b2YyMP|q*qyCHnwHH8eKamS^~41z5KG^FO^w9N_q#|&M{$WX z5|h;HaDhQbXR95#+K)}Z=^jKVBdbT+*iVkQg^!Lx~`k9#tLPypVk& zZ*iAF7j=FRk9ul{tA!aoAzWXB-WJ6=s(=SJ_MQCvEQyq2M@K}nPjJ2iOlT6GgOq<7 z*DEo#hJAc@g76DUfycBW_HoQFE?q{yG?smg3xiD0p~OXBV>L$eC{B-69la-)oy6CJ zO5g$XW8r-2ta7Imt2*YnSYssS*j!aU&*f9I`jBGufXCNysA6%30|;jm_*9&aiz-t6 zF*64lBVc4`D&u4Hx4wMp^WEv?p$Aod^WanE)Si5!#>_7@M%3HLrjEWjh)-R0ww{!m z7x>gI+w0-D=f+yJeF`v5hk+2x5Y3i68mz3wQQbfL$Odp<9;?UB;z*7COPuyoZK72%g z`l#}IC7t=O`LYTKUghiq**}#ysKkr-ie)-OdtB$S?8S;t zUG^AT8_OfH7ELAaK_rCMfdd&(73<6m3PZ1TG=Z90BkjdB3P zCgLH1ZA)S=VxA5Jscp1>jUI{&04gDvM)ywAhhP~DKcusxlY{j1`lp;l=f<;{NRESI zW{G=)48Lbai{U-YsHqyd*00NjAu3^2C-e)pZ^n0D$9pSqObEKBD{`GswjUG1F6qPS zB`m8GX3S$rm|%t!v&uWA`fu{=Faigr*_ZTmbkD(1;+TNG)LQiUsR|xVov?f>TZ<&{ zg*ax^pMXtR)tPPlwOq7?-~qz_Li8~h^5BQbS#2 z5H?k^kT)g}1{n?$x4E$pykf8?NkWOoBlwG0N&`dbxrvh)Pn5T*nN=>F($`}Nyu=wF z6Orn*4jZ?KAgn&g%}rm2jgBrJSWH2pjw>7{{#MP_!M8vu#ZC$gQx0Gi5Ckc9QeR)? zAchHbR-ZJjRM~~S)pbZG4RmK~k!L`<;89gUK+u^MQ`Lz-r1159#f>>6T2m*Ts9`52 z%R>@`k)Q2_(+RKMb0P(g8_Ddlx%vjC<3Mm{mp_|Hs?(#Y&u%PaRxB?NFes?0&whK- zvJ4S)8|q&rtH(AfwxpUzatmP3e&NUd!e<^oB#visy4Y7Z@t{t**mjjHZc%SqDp7P@LVm${fL z{`W+DbR=z`+k2Q>ew2BifUm00?cd5RIJ^@`Nn@)zc}yCc4ZjdURGGZGnzc7i;skt6 zoxE)jd;90^0(z7nWIbdXWCDaV;pB^4rETH`W2G#9(L= zdM7E*)}LUG3y4sr;#Ic+2OjXiK^&9npPn5zwSql{>>nxt=$blpp9c@Tn@t4Cbn*f_ zU+wWiJ+WJSX%{CIm&(N*KdHQj8x2}Ic})Ia2w&j-xUiUDIt5+Yx3R>DACl>mmf>uh znPmbKTGc77{aNmBw}n^`G0QJez281h8puWGmB<9R^#-64Ta(D%;G&dRb!a zeI*N0Odddr6B09VJv1ax73lKNosY(be4EqR(>_RHZ=aW!luKPOh4gk*z2$}YNzIK+ z9DYLvLN!f|?^eUwV{n%0Z0O_t(^z|SH^>brVO8Uo6zP>1H&B&C8k@#{QfOr2lSBl7 z1CNicC_iEG0RNiy_}?+U>g+%fFhHuQ)5n(THRkdp;9J${MZs)!6Yi=q{m15P6-GGt zm;9osGm4HV$>wUc;4$O-9OZxbs#6I;Nbrm_o`hI}F0`F-HIcPP71LtiaW$CD#Bwc_ zkWA48&o&Qc1rJr7fsfz9cc8K1=R8BOKAZ@>azTki9e>m8hd3#Laam5;FM1yB?c zRyEjSKh+CWkkI2+Irha$WG)NoV^v@DiDvV$gawh(l`mT2*d#Nf3d(8fi?0^xv(bekivb4_n(MP%NByYk zi)(V&*8BAiG6L}vcFr{C0qM=h4Rh{{4hH@K8tt1vDtm@3vyX-icEr60z zW)1hYm!pec1tUWrO`V1Lk7JE}MXZCUsk0YE+b@|DM6fGLxLx3@u)1P{8w6qYD>W6{&zmdp|B4csB=%KxELZmL5d?wsm2QdLhHo|)%AlL$@6Fz< zp9?n!YxCL2J$IEDTFVx|`v65&)6_Z7r&unSr_NRh0?T4=Ak~XXl#xB=@KbC)j-zu` z!OYd0^{*$GGd_W>#i?&lo&16-@wse)dHNiPR&@@vO)(d^QR4qa$OvL7?YEOiauiBr zEN15<2I;vu8CmQcPR<7odLZRq5_=WNf+Q9X2!D@duV8t>B!I_%3AOAMbKfhZXlk-& zB%6%B0~ts#O@90^TZ{f_9gbwnCbkaK9d%-pN88w%>GDQ;p>1+pF`Hwi&q=1!gV=h6 zHljz`BYDCZwlVz>{g5_IP6%N(WHOOh5o5`*UTht{gj7Z1m{iKvHR~jBWRkYY&j&D@ z{Ug_locy9cTZhj)@sK!>9kvcbb6_x~V1awBJRZj^@v1*tjTuFhkc3yiIK|g?O?GMv zDJx6anxg!Ag!P-)2IR8ArY6(*SJ(!O3G^nV)NIa{|FyRBlv~)NIRxGpJYMrZ z!mOCIrz*lgujLlA?&z-rm?@=OHeVAXQ{Pf4=#pQ?TNx~po@JG}&unE~u^;(roz=^u z`I?LJa96=}?%Nlbj}$4H&OLL9J%tI?I$(2aAM&}+APUr3OV+%0gr4mrzs?N=E z<8StS;F?33Hz2r1$6kWap`KMMsgOsOsyhtL+AMl%c-lF|TC+UsfXz1A@ky zI`>KpU-hM&#TUUp_rXKH>V1(HOiVR(UawNV>cx5!%x~b7)pwDtPrx8WQ|HgLGY>e5 zaNx133-*WeAsf7e2~kQ@7nEl4!55CZ1|th@gz|86L%$IF=FwV~WlmK9n5r%uRLWO! zKiAjv&EYkCu>0R4q=Zf1Y?{PpFKFg`O*Qq+6%~9Ly0m2grW3G~VcC52(7CQEDQqzx z4JlAeDwkq!&SxFEA^PipuBnUM$MUz{xatb+#XE{vj+P^ogBdk-ahtn5uJb!$Tq&w7 z3C6AElbg7tESYkjFGgqsB~{bZ)UDOZQlv68fWiL`kzq{pf*9Xp@DQ230%Axfm4s^)K> z6}N|(Oy4ZzFU;5Jhm5hM4}BRf8Wl)|8&Yqb=m?T;`%yUpjW*E9`tpR;~B=On0V+Dq7n zgAKWWP3}aMq)U7qcKMUV1Yy}uFW$e|9f4G6yKMI{-XDvPghPm3hMb9h*xHo|dkA35 zz3*}ykfH!K@zK=f&!+ICMajbI&_`3#E3UB&Oqrm*RBna4Kiee%d*L!rHD%zAduT;>o=SIr#6+jta-K{J4%O2$cc2J;^PMvtnlnitJmW62jdWOI?x z`JE*K`#y~G4JH97dg?&lbn*@SK5ub{Uuq94lt8ZK~f^dH=FtF~}%A~H| z@5Z}*-$cBGLjBc75cmE5C(=bqS@Ttmq2ts8HyKia>YAao+=|;wuEM5Ztf^};zHOxb zEqFwcD>qC`WO+CM$E-3|-LSNPPh8?Jj4ZU>aMqniq{KNz7P@!)}{^aGP*!yB2N$~s%(u?awlLbECre~1xyI(EavccpH8peeVI>qv@u^T>_$4#_ zlCcrX3pNCd0G9b_ih;w#2$+PP)g;YmgVY2aRfK`EIvru%a9b836+E(h<5*Wr+)<1P zDOqoL9Nmvbnv@ev-yN1{WVcyDKP1z4PhTVcm%y^ScpA%5f(U|XcDGZ;GVIiW zktL4ouqNDJ2_;_4j5xbZ(c^)a+FkjA_^(1qm{wve#xAFQN-a%0VBL$cd*D z%3Q5O_MS7wMBH(hiYjH~{p`Y0VNWlkUkB1r){l)1GVUouQ4`V`H0M$23{D+6# z$B4$A2PjUDRn7h(+<41O)c{kgx_wxh(FePI^5H0$QB${%tu&VA{41)U1PEJe*a%m` zG6)wclwC;GAgc?DZLdjJcA~Fc=j`qOcq;ADVKsqS)g72&9e^W;8WN>A+N zvB{PWCBg?;{%0<739&n@zKSLHfs==(?ihYVX(7);kp|lFVvf=hDH3HeJ_f2eO{4Yq zuuG3{0Gw6LY38e!#9kJ@MJY|qX;*4l^K0F@7zmFiSiZ;YlyxS|@vT&rU=f3mf`_W+ z6h$hl&DDEhpd5XxGJNpQq6(@(%9V1Z@1HxI1XQ_o-M-D7b%OUGCESf|!5(iNOaTsr zJ6`s+t{y!I|AgBQnrGUh%1Cu*XJ%h@eUH;~nz}Qj*4T@DZQ=o|S=F6S#@hG)*g%vJ zICc&#wLgxvHE1fsVdwI23(N$Ur35DVKX@u}-(i69GsyHTSDXeK;~zfYKzCW{Eiz)R(TzYq80Q zMrh|d4U}i8Rcu&os#7j+TAb0%+}k9W=8ZeUwxd3@C41mD`5E4ZFJ3X|K_g?_Q5bO{EbV=_3fJ zf)aPUjAyah1Psy*`P^H1xwkz21Vr4go}Ohyp>d=*g~sF5`~%mGL!ZrbwjU^QG|uRO znQt?aaTWSswMru}wSg(n;->G`^NqsSVrkAmt^^dcPc{nCJ%`1}NGRcT*!cfAd;2h( zsy6T+0ANJ%<_NtvQX16h$hfQXVShnUqRRDay-~CrMOF2{o0> z>@7-0ROBtg5UG?JPl{5%&suArIp_QMUf1`!et+yg&N}z|z1G@muXV5cE@=0(d5anK zm;aEenqASzSe*tSu0P?f;5^EglsevJs{_ky-axGnWtmeG-Px5kY7fFRES*xaAQGfU zdFYa(r4$~Cy!g(q!c5lthHO)!sU}LT){T{Jd#-lTV4eN?d4*RGkpGnXmA?8^$t<4T z|1DmPC}v0Y%J0{?Irh8K_MkV;tH#*A(2Lz$6hLe}h$zsPoE&&A`UU((Zid`tGp1O< z-Z)zBUg)2i8%f_dHo?{9#I)$fVsHF+Tj}Mz#j139v*nc1q3p6?LJRTcMMq1AFn<)< zS)1x|!@9d8qsPme?F-eN94!{0J`Z)@7FKfykyk;}Fl^s^d6qoh^n25j@h>zu^!w@J zvu05P3T1c_M+e>~y+sw5RavX$zY}jT7Q<>lEdSrKMCVm9+Oyj2z#2x=FmkN=DKD^| zE1Afl=}=arWq$v1w$zZpc8Bwe%H(B9Pr5c@TQ{6vW#<1~-DT};`4<8H_|~WKMUtE4 zw2Kg3b97zl!;FT#jBF#=+g)PanZ$lo3;X5mQ1?Dwg*{fzrAZ* zX*Q`#P~Qr+x_+@*kTXl$)IzK#bHk$O1?%e9rvwU+LvukmN-eBeUpk)sG1S*INXMVB zF|dG`D}LDS@XqB$CPA$3$p+<}jBye;J3PP-zv1S6?i50J5!)*F&KG%sn(RqK!;Y7C zxIzt%u&9@?Vo{Uz6M3hLBq01GBEN+rs1PlcPw&L2}W}H(r<}_MHh_*M8%H&-;lVbAne;U zN2Ue#`|A%&^nbg0vJ`7&r89&Sv>cJ`=PFVIAtf^s6PM zM?Ps~wp-kT7kUSDo#2t-j_GXp=G7D`4eV<`4D2=tOpD zpSNe?=o1Qz3G7n=4*pFkK!)`K580<=zmDPxS^|=>5TUS zOENR%ZPwZe{wg@|%B&<=3tOK?D6ry|9kPn~R5$R10E4Q0hwCROXgRJIdwuX#RUr3kg{=%21b_1+K(kLgsv=^&S?R_n& z@LlSD_V)pOCsF$*BuzP8De;3AH5X|~l4_4ba(-wu8c3Dy@v_h+EY%IIs;_e!`>$6+ zqZKzNOX*_;{&yQ8z3 zCl6c|*poP{YTAlMdK zAL5W42rF3EjLD@B`L9CJI8RGz`iGpCUi|XnT=!F3)6ZJHXZu2TC%Y_I z+$xvW;;6fw3%T0+unkG?HpQ;j*OZ&HD1Av*Sxf?*YW^UUj6Un7oQ$uCZThM7CD^!_ zKv3{Clq3B=&Xjllnjh|CjwrGNt;fh~!jk6fDRI}^%|FX!`0Xd%b(|ETnAIik^O=ES zMgt9|C(G3_6TN=y!qH(Y#MQ~k-W3}TXq#HmN9`$*H!ic)kB#R>T(sM}c~Y64U@ezX z#GM!On%6|D>HaaQYn=NbM_7gaOG`$R)qFiu4!T$_*9}i|UEavrKxiD2F%(ir;$`9Y zkD!b@TjBLs9x47T6K3Ri4ps+TOB<3~)tvs#nfjoipO8#i9OKoy^5p)RPbj_GHV<^a@qe+hdGO9|=l9dL3^tp~SGX zzJ0K~m;l{tBaSL~YNj{1U6vtg3`AyhRUOg6xAu6Xq=Tx-Z6B?iG0-Diw;e+OH!@f^ zJY2fZOFw3`wf1Ki>8qKt}Y( zMtAahjlI~+9|mIfb9wH!go+jd!4s6ni>3rFj_lGp^4OfwUZ0wo#{1A9A^QW9z1|3> z*%}&8-o<$XvG{|nZ-es0-XbsU{SA7rPfsX8c`Bhuo*Zl(t#zTHO|d-5p)dsX6O=jq za#XEbZLl$PLh=-;uupN@crO;i&>?yH{S;Npo{U%n%_-bg(uppm4%vpQ9q#dye zz8sD`ef+fC?_VD@Y@g1WZLU%Sp-~|EzYv_g$aFq=bcZp1nLB;GN__kg&C+)5c{eAj zng{Kr2}BRcoYgbDG4_PlLOe}bvus{Th;1c(?r^gF!*a9`0~wV00g`8N!%oWq;gHN9 zzgm8sW7EGHhy|o?WY1A=xa%h*&(i5?P^KvXd+>y0!3vQ-?2TpHKNenbTq;?gg0F27 zl!dqKP&MTu&B5w&9*q~tbX1){Pz1A`fAP(U?qd`FFuf2OAz8fXjH<~kEQngsm*~+z z5IFtRP6mrfzV`@Y|5SL&SxgCxM{Y9}*sWma<+Oi^TE0Ko8OE}G>4|t%v+MFOm!K@= z2)r&402s)!eTnPomE@uWdMaf7ma*=kjbgAgq`W^`ovFUwTTt3Ec8(h!sti;7guUb&_x z8(mf`pP*_kvG?YIARRyRN3A83!YCnmEqAl3`O8n?hM>IKbV-T6%%Zs(r7jNGowHY{FGn#>#CFEesFBH)rIOj%AUj zYulFLtObyrlw(L%3Ld9gWb#R5o=-#a>AQ>F@0c!1p@BXj`D{m;)TJ;Dwy;D$Z&BZ?TRPW37%rcaT)t7S zZ}nL@UER;L$dq6 z3a|Ffwkne$x_fthcX#qyBPn(b%Ko3@yfL?gvhj*Ka zzDx^t;PyEBX}7Q|gF@hfSA%#5{not-XK;^Q|ZDj*v z?M(O0**acbwxFxzICx~Y`^aE3AKN~5_`W)_j&H&$h>AbgOAgZ>XJxqUiXrg@k{SMO$5zdhxI0FCH&nI1`2hjKUq=5XET67B$ zS|T?y2$91Y=!cZC8%{Dw>joe~^1}_u8!EYcuKO4b!{vMOT0e}oenUup)K*K=%_mQ2FL_BUa8hAqT+obfs@3?_~_2bANO*YHvSIu=OYq8&dpCqmcsCG0h2(EOthRE4YA-VkoPw%G0)FuZ2DC1NflzgOE><-b zmY7ZlYZ~&4RE^h9nJxv5psL$qT4325zZ#+z<$sgh*ZDftJ@fn_Gi?=WqV@JwsOpqN zZRn3ERqv&BlEpQo0)Nb@dT*!6Kz|>G#*k`IbG52RL?-y!1R-_7>~X3_mvrMwq@|%? zlH6~ryI6?Wsf*P;O~046Oe{P>)o9BOul@A(hQoZJ>2`3CYN;Sh#@3);~XbfgVsbu+1fi_OT9Pg~l{KAa%R|C`=AZ}tThGRXiHG(DTtw$z>xNd`ZXxu(>NT%gO`aj6 zDa)+=f*C`q&Emd2R$r} zwq)Tg;(A9Zg3&l!wdH*~-k+r5;;8dzw4=7*LaUq>jtu^!`l%pwnxqRxq#9nU#|9# zNkMZ|$2}z_ncQjCk^%(;l}K*kaJowUPHg|29sa5DKCFRgoaUY(;!d-*a?IJ{o_Y95 zv)bCLCGFC>T(%NQTm^9{pXRkjchc`lYoM-sa;0IUQjMd!yqf6MrRcIo!LHOA;Qev| zM--*F0pUWbYsEVEK3>3DQVUUDEOC4UXb8|kDmA`Dnm4&3V#02hp9p;5pC%YdFaI{P zv^jAo+H+-^Xd0Vt3GC(gL}_4sny_(=Hn4sim45AH)tG?x0=R$%#IG~F)~x8)*l`i$tpv&~pLU0pQyfr#hMt45H% zveJv0a@>e+Q2Ji5q7>>8<^F$^C&)OHB8yhquoGjTe(dzVuy0CuREAVvc4~_8+$k&u zw~*>LV1m4G@Mww~2ZLGY&YaXGj`Wb9hd`5gwY1sRHD%N=;3Dkju?LqxE?sij% zECHNzH^yoX;GXXXQ{aV+-bNNwwd);s<^v1MI4oR#c>t) z?p4Mw20|{*^al06T@MW{7!?hAZ1e^V``Mu2EQ%n+>~??SrG-!I0Lo|-`N_XP}6XlZS<}guru;HaX`5y`wb%)XdxBM%kye`d(3EnH8l^4OT0l`wMRK? zpn>zcd({aC!lqWafw#tc1IXQz&j7M)55#zbK7KMX_OpUXUS?;TWW^92bj5V^Gz1)2t83<59YVdp0yg{cML{J7FPWA>x z#gM2WZL4`h37*uxH7G-1TLyY=mn#+~JGhXh4CW^p1`^sy}yB3LpNRLedB13j(LmXVoyWcY3?QVmqUSPR`u}Ltju<&YP4d0$4?F8#=+pRRnRnHL z7a1C$g;XB3NduB47tZ~r`7g&tn*4O?8Ur*Sc#VmQ{)xw^R9aPC!?H^Bd_aB?*>0I)Mz`R zQJgmskH|Tk5WJ6M5CNfv~sh%_MwSp`rNB^x#sIsukTQM=4ptIo}cRVxNU6YWik5lSZ{RS z`Nqg)x))Nc)EnKaURZQTjlQChH@f$~q8hq)_eQgK0YUvZ>e}OpUJk3RLfdhJYV4*n z-q84|T1t%;Qsd&Ld$mc$(3bD)hPt0#|6{y2lz0aX0gaHFF!ZF9y;A7ZVI9SjtFCXi z&btv;@~?#$dtkD6TcPR^n=6Jz3(~Vyew4{?>ovoU7Fhew>%%I#;;$2vNr+M+w5LCC0 zo#a-KJ)0h6iJsSR%e*C?bNZNZ6zXTIoBqi4?x550!wPoukCom;76P!eqv6)k^Sv90 z5zvAeXWyQ>%xm|Q{Rl8}O#Ww{yyQoA4c3C)_DYgBiIt;9G2LMjZU`mW{KR;5%;7?b^(=42YrmOS0s4g0jI&d`{E<^5Hk~&4g!jUOdn0lbTyfmH^Q)2Jz8+FDf1eh3 zkU%cFgSBbFe>1%?l#1*f?zl74j=JaA{aPM2P(fv~cLTQuO6{tTS~htb-TUaLh;D8A zQ1NN^zSGCF>iVJl&4-oV7+!0O!bBhXc(pf*oIB`Y+0Gg|)9X0q8{;_|L+WAr$4#8g zYpWaeXSJE>UG;ozLlin8HEW=FSNRD+j0qo|R9^Z%!7az|9MXo=V;98qk0llk87!Aa zWQI55)TW3WkDlyMTI{cbfC{Nc9t2{_h_D<%^~lr9ysMc|AgHCK9?o@f-qk0@M!f$h z?}q%>6Ld-l>+DBAp6-oUP(Px6AsaH=ercjZ8ugQ_9=o;DTM%asQmt}Nb9}v$gF`r2 z6FzI(XKpVpec^GlFtTzyv$v>J z7gE?8w)!)_Z7ltd{eH1*pE$bFTf{9rEM{pu+cDR@f8!c+v{aUVz-Yf@wY5mF29?p}JF5l#HMuB?)) z7Tq@AUHWmRF(Dlv3y9atedMIclQRK#!&;bw##4WUa=e!r9&ZE?ulp%hOz3pmYwr{PN<|Sl(VFt31<86wzKQ-7yWmreO^XoeI zeLl``rREe;YwKrtb>6gb(x41iYoCdAKe~ReIa)lWZTjx=RJVM8ZG(cdLTdF#>E5E< zmxW^{Lh6IUZ8C?5e+1DFb!hsmcURWfV^)ra%X@ch@#cSEZ)(y7f?(A{Ky+r<&C5#)M zpxU(~P1U%-UNbQU+WYZw_29lsO%E#6qjL9aTy*^1Y<5{U?0N2tEa2c!bE&3#;ocV0 zWWk)#`iE^ev@S_KEqgV%mE*wDd{zCd$%kU)IB-p+ypd>vjqUt#VDwm7>E~uB(`?U! zICN^aYWBHJjAqzYv`-R`_idH7hhNW-zoODMtHbZ)%b%RzmLeEXA$6D{kY|ayK`}cj zkHoE(KiOqrGSH1db);jioPB4hmeNXkq*IZcU3**)K>axCNQ+fc;^*xF3Y4jf?PMoyD%UjxArT>YlJ!G>tCbPsx<8iC_lB()i)TSXGDU(yE3NTtD7VTuwol z=+A9()MVbbh4{tL__-oW)#zJGJIZQ%^4=Y)#@!~tfpz7{C(C6&Cq+P5N1d$PtZH@Y zAMx!k6EozSYIF6!mc}oXuGu){MbiKp6>~nHDH|X9Q=2cNOgH>G1fm>sXnPoo{XB{F zX+gHW8)fcCsmf{X&R3s`A51b>S9!QQDaB z6pqUBHxO;ZoDQlnuSS08=l@n9=bqWJZTj8Xp4QpF%{&)ait5^gJVAAO`Y;t6v&Kxp zKv5v)T)FD`+kLu3thLzbpX1a8XHpCeAcE?5F7VHoA4PIz3|D`U7;R-%!01v<3--tLv9h_J-OR8wrZf_l zb7VBN{o_nbciD8rY{~l#7nIB9>+Xzn?KA9-7V}zzpmo%l&UNJDUb#jNh&t-bOIfmc zq^Y~2dq7B?d2610L{<&l=o5Wuh7`@TIjN2MXFi@TE7^jk!)aMU>g+YMo7seS)OaQ6ZAZGUc+C_^4$4X95?T| zV+P{p*|L=EFMwN=zb_jtr#QqwC@bl?PCMMI$cuj}Y@ol>R>>*8J4W4aWqDQwmDw9K{J9hOE6d08s1EjNzBNI@s&lMYhBjnqtLm-Og%mUwGa8(lE%RmWvRkSq6sKnE z7+KhHe3W2q(_~@(m8Jp0(D+xL%pciR%b=g2Q*+-+ne)`vs_3P40&l|J0pkTQoCD)YW=7U(ksv?wUAi*=%Kq64eMg$OWxUlYlxszcib>{ z5rKXT9?dD})a!U!)!;Y|JuK0B51x>BiT?4~0x|QPyi3>#_|~TNW=)oNxc~x$et;ld z;jzWfnVyVIsi)O5Reohc@r_Kw5TXc(+CvVFLc{grI`vO+%e^RggI(5ZhePd@xKU<9)eXOm?G$6#=;#ua0T3f|ZO`4g8wZ&T^r?Up4A>~ch{M5Sg(u`(&?}LKH;p|o5Jk(}8D90WhXeC*c7e9~ z1)LIpz}SzV42dZvoX2!JOXKDBveFN*Gm>7g=d_$6$`yio4uQtjN+}MWiP+ zE_5p8Br%GmbSYzx3%7Sr^|@SJ2wQ`4;jk@U3)mv1CEB>jO4W?mK!oxe&MThw=utW3 zr$8_!Z2Z^fM-mdB-L=)EfL?wGSnM>H8PAsX|#c(Wgk~EQB0eF1~R7XBg2rkhY_p^C%k`i zR}_0_ZL^&wTpX`mH*&h-H2Hag7sGbP@-Rx%{2gA*HN(uNJ<^7p^BW45c`>tFnlC;J z@xKjm+?#Jk^s&~(XdAa?mTF#VTwtvp*E7zm&G7>G*86dPPf&3rcP+E(J5Jo+#j0+5 zN83WvpsZQ56|(n>6Q(DFHr#18yhPPGK3?PdG`7s}Y8Lh}AKs_*@QC}Yj#rafdRjRn zN3(J9s`m8G#tm?yidoSd6?b!0v2?gN@m}>ht;`3nF=6xCCsq90KN&fY1(eM*-Mt)O z9)qZ*ad4XZE|EJ8=~6UO*6jLN701mstsD<1e@=8y{MsY%4G^ZGd996J-S2HuGt(`b zw<~hbung2hkv8ZwAG5`)d$);9wEr`!IVo z+vb0b^%|rvjLfGkVq@L2$4pWIYq1uqcgUYCC3O#iFxAxV?(RO}FY>T!$Z7FgxvHo3 zM21*=Lab_Wt-bROO2~<)GHr{oleCkpqvDh1saiMO8tJ$#HsyJ>8k$XeD|-CH+fOtUj6TPL?Bw$ zPF3-@pA7fHkdyG%Xs=#=-N?cy;q6?nE+3kRZO2^8rn|k`B+wb)N8LCrx5VsVbDno#mm7vmzf4*k3_Ch9*o>22IAS( zs)?U0%5X`1V54m6S0geRB$6Jd$%Lo`^@-HP_;|nF_+O-7Yl-9Md$qYl0teea5^tn5 zeC#L_I-8;;Z8dO}9Af`;0{zr-Ih5vIz~*o%v!&6cb&I3wv0Zr*BHH#1gOXq5)nIwf zXtF4+X6*E07Mox}Yq8eVXQ&viGbDs(?N)ozygCm}G(-_B^EkKE9xd9gofKOnyIBVnVnQCu zymd-Z4Y^AUxYf4Jr^~#qo0S>rgAGLE({4^t;2zCi2bUxaVrcty$ zKpf8%VHzK7C+VQm@siOZgz;eutsrI12INRO$|0qGfEct{di&v~7UH^8xrERrjBLkC z$MKnx=I6_?5L1pz0@H=&!UK(IBAsF;8cB`SJ5KK|Eu*i=cbru&mqlOe?D*mqN%=pB zF4-sjxFnDeUZHe+HBqwub$Pfuz~${_auNO17ad(z&ya3ZVTPqI={s|z8wIv}2*>I8 z-ch;MPYO`Rjx!Cb#M$*-m?$*f+ad$m5bqFw=-+V@MxO)F#o zr(O}8Zpo5sT32}=8s#}cx!W)e8p!vV&+MQ(E_5LJ<1Fc83tL%NcKo?mMssbCe{GlQ z`145-{}!ip)Zq;G252m&(^(q-Oq0Hk-WcW*ayr!PvsCz7?z!k#2!V`K+MRDY3h>B5_`oww-#P5SgE9jBMoSB(ZX(y*{X+ zGM!EoxERI<9zU#dok)u}jCf1mw~o_k+%QR@vbA;;G-$XbRz}o(GD@(Cb!0@$ME$Q1 zarbVy#!nz@?9plJOzFo_i59F(`?k})>m+dK;xJJ}zkjt{e)!gK0|w%Hdh_QGg^2>e zcjc4xa|F{5qCeb822v3Pf*J~xC8wnY+WHNkEZZhIbbi?Chlb)bX-NVTc`@=M_$I&w)1f=?d5TZZtlrdcI(Arv} zr{X1m_aj_REs;wJzm3RoE?q`o4=p+m#RfT=&?u1xl6Q`HqB0^_=L^zhfS=~vaOs>d zP6pawQY&fa%topntDJ)HgwlEBY*mjh94#pi)dz==3pz4hU`w*>hP8JZNf8c++?CRV zi{t$8iH3H|B^TckCC8-6s$SG)xkKSZ89^|I9zX~NjR`xY&z5mvqEx@RqrNnZBoBw} zooQ0f`qoF88Ye9un;pgF(W9!~G?M{S+ZCcSljZtzX2Z-nsxt-Y+C(SJ?L2di+(2#@ z?FOsMgA`7Tc4g<88B*Jau#yt`a+b6Cgz&`H`KdVZX1@{Th9?VU5H+X(_leF;mK&I* zd;_7&ym-msqd8)+c}wI=F!FHI3WLUeJtWDR>Is)3}4S=|~(H>{LAVw`#D zX^k;5d%*$Xf;9~q@EirD@~mGp8W3xSNitbiv^{uWyJn&!-|?Wa7!bpq&g;%dka8Xo zi+va`S2HIUGSk5c8muiF5iS;SR9U86$*loc`doJ8$_NfYH3W^Ie7RCakZVu(F}TpM zbF7SFbwPvQwZEAzIb7c0U*BTiFd1ChH%g9gGbR527<0#~$xxe<$wxVnDLwsqz?Mtp z6lsxGY6L?L>`}Q{TBMAPNP20_Oi>Xol-{hnPnuydz%^^Cb=EyC&B&I-56k7!<`blk z|9Z6$T!bA!miH>rFQpX6wF^QK(JyVaO&mhat1NbDo0F1ELNeU|4T^-e&D9WEef=Ou z`xVj}Cn1N=mP|(>@;Yf}1A$v_Q?|)O|D4d$xMP(x zVUHegU)$+Rq}Tnmq9o;V&lL%EjV?%vU?iDs8F@$1@Jx{m-oGHqrt@Y=XL3|Uh`umi zhL&`Qvd7}daxHJ6|ZU2dK0~>eBUu{{7 zomegHqplx2m;RC_V=vnhg+c+gE_TV{`-QTIBfc1Al*66lolDvKY{>^6GWZbZsDk)w z4`U!2Go4FmNDJ%>_Ydm5R+}VQ_uC3WMs(fKjX)i(oUum=^$D+IQG=A0XiB4*GTe4t zO?bSM!pT`Iu}<1zMwgTer%BJ~Oui|NPD(3ZQXgWAG$ELyAiNArX*ov{t6vn>cDR!g zJI)=-Sv26*4JqwnC6+8S$HM(Ig>#@$tjCZ+qd86r8B@km9$KUD1f7(AJ5}P%<|IH1 zM?=?CN#@<)qo98Oc@p&BK%meNm?W|OM>9~MkkUVcjc(of3-xwbO5Pc%$DI+tt@l$1 zTbs_xkC2l9y$FVM9nl;LeqarM~;}Wdsu)VpO40(0D5e$e- z8cHPTzKVWps%%Pn;Kn3Vr#opVQ0`;aw%4GwUts%miFCyt2ir zuEe*#EB~|8y^@VgtsGDgWf~u}yM9;>8rEh?CtKxC)1jfbL`HCk;S<#jr)5OOMIrjwfFWPo)5Y#7UKR*twe7;v>1yI}6T-7p7XlzI zOHS@bgGK2=ZvAl)&xM>W?KjG}rX$1ZQ@Qwx>2fjgxK)1X(mP9X`8e|pMAhM;ajIVQ zw?~&TtYMrr{R_h{zAn@jRFQgG*-jT0M(zBSqcLHZCks`*K(=<2HUV;Qvn`L!j0oHH zt7Z5_!7y6{J60^CDzdb4wx4$SPGr>9*CQtEa&)7Niqs^hwlOuz+H$r=|F8z?+Hjj> z`067^rqi`cjJoEAWJANsQ8~rCKdzh38)`ymoUWXV4D~NefkKVtG4oV)igi>;+La94 zXW_!|u!41EukQh_%Iml-=mee1Zq8LLh?Uf$gGO!ReRX6phr)~ojYb*bCK;;MO^+Kn z7U|KDnn>kh!jbu*#it>dCokBPcDliFQgb)U*JQ!d5C{guP1EEoSGxIOZJIhMN!46j zZX`9zrLv<~ll>sg*2t0iT#RZ+dW$MuQvbV6&Ted{t!_Ka<@q~GU*=9?X?TUw?ZG1V zg(sU=1++VtKe^F+aKj}gP6;|xF8-iGPP3t^t&Va*r(4_E@=A0Cp>FN6N~ayN^^%O} z-I!s1dv?3#WvbdGrow|YQ1{+DRJEnIM*2thvDLhJ*A*L^7HD0%{gkJw z`D5SsrTbKl^QqheOD!$ck3U(ZMUPC5-M66&C-2}}BX`H9r#*A>Zgsh0UxrGIerff% zIJ5sT0zs=#dQej*)sAY-CFJzrB(bxkGr^mG$ zC8z&;5u5h7E7?6rk>CJ=SsHh*cTaA9LL121qsIeza+dqqSWEW+*gmsE)o%Hb=~7mX zkEg3zoB^Z0Rj$WjHq6ikaI0JozC1&0|7cxI$L+CQ++TX!7daN}ai|0?zl8-0Iz4_D z$>-GwxG%@KB)5_U$tn8jG2Qul)=!oW7ycZWO&T5v(%@@|Y^NvT zgh87b-KyV{ucV78Orza^ZxHO2Gt$>zC|X_Km@j#J;30<(K^V?e9O;2#$8FC~b7j0= zC&04(Y^CJ;S0RCdD2uCcgzzGR6+NxtdsBgmi|sRrGwT(}(@F;E&c zF?!e)z3mRVA=7%_cHFd!x5?#x-VLK|T9-l@9BtFI%d?~l*G!hsVfGp>-SVZat^4K+ z)_sYLq2v_K_5~w6YAB}(n9%1kY@Q7BPeqM^(#B1coBZ{yeY9w0llVE+M=P(Y8=qJ?-vUk`sAXQiFTiG#TfA0b8PTrpZV@l^hK~P?1gi4=Zai zb_1^`n!$JFa3}49&2lySQh;mmvz;`LWeA;Yg79MD7yjp)ol6L2Lze2aaNC0E-S#<%H?|0y4ywx>jTu?q)5 z-=@2gB$D_$iH|X!KbkqiPI{k~ICC zlyIS66SXL5TjXZ$crU6Fj7`YqtdlhajX|f^p!~qG#9@I)P#+V(CHsU7WVSrTxCMfS z;0nnk|5&ND2PkY)_9tDkrhz^*Tr(^1HB(<(AlB${+lw>47pJBMZo{{G_59I`#rFU_ z2T_ei17&Br@YM*o^?t8OlVt=)Hr>P1MX%dWN-lZfB5t6bc(X-^`!@`Cx?XHl)n-ux zgfA(hVMqTCS`Xh151a`cB{;t+T&yntJ{s6b0475Dl0Fb4pOSM}(?AzQe`2@9CWuR> zk_Zj=gp47=%r_z3fyrPDH>b567KJFreK*`1?l7?ZEMGp0)QhHJTe@s}uWECLhA%TD zV0WE-1ASE~0Y2U%f)U;RXCyHRD0_EEChM`mbUIkmP(C4W=IMVM^=YKx@HuHt8Ww)| zf*s3ozvyUVt{{x)-$~$&k`7Uh`nHpl5mta6+8#jsk`^c>|0&>FABdjpAWhim<%cgR zJI(|AOaq_+@kfcIlMYuK2(*yXn;Odpk@6I-EYuLa;~GgkuYxr+7}0xk0CDgddz;ks zkKWCTrHl^=@U0xZ*_Uem*&{|U!=*PnS7k&pA*H4fbb7Z;mhvZU8cEP-XqzZKqW3L( zCry)!cq4;i?a}+9S<;&CeJv@C2-dcmlyNI9($FzYzQ}#R$YF`59G4$>HsstG_3kmGP;m8KM=m79$F$5_nXV3nk~vbmL?lcMO85DO@+%H z1aKb>pL1<0*)CBn|Vh?CMc}Bi!X$~KK-#&XhaDeO{S`HqRd%BuzXU0Vr zA7ViV=|s3`L6!Q?O_tAWHKZ_&ZE}nlG^QNi4a+Bpifrr-cY4286xc;x7A>iD6wA@l zx3)x?@bx<4v4s>dVR3F?7oR+^^$o=3jRStzJVrv7Rf}aaGcLx_bZGcsj%3?&Q(v(6 z=E)(-yJ8Fdpxj3rWh4VxqtE~f;e_pY%|~c_R4zv_GE(|PH}902*e&)A1Paw$`;gZO z88i)i`w3f}k@*8I1cC4F-yy6V4O`>o3vMiH8lcfo)=0{uD_i&8mLwC>O*(Dg*M7ipLl+R@*93DmkPqw@vV>Y`)DbfQdONH=%iOm3!JfKsx?Hm zlU}PlP(eV*BDnB`obiqS@-fB1A?kDKbV5E^a5T&oh>Uc1N52|T+GdaQl3Gp)K+rBo(!0-+Kx7LV6dIn1 z9zf~6W911ZM^x8@heNbbHgb!?R}K_nHy-1KQ)nTO)WO--TE1YxB-zgQTtSsz*kroF zF98Z+Xbd_h+w0sI)`f=QX_7-i5Ky$mXc#4HWY&f^jqCcB6mUlX2_)AI`D2inAhz}#4HG8F7sG8b4=d>{ngN<_ct_%E1rp2a63Q+``R2Vo$;A4K3`crpGiJz&&LfpfoS2MA4JWwueiEzha`auqkpT6PJl}6zi=%nA@NKO;VQ;1-e=&TizflnalTb($A z&xqXdxj^IbWI2M70oTuXC;iz>`J!Ic$*SX|KNllk`>Rjrpv(MX`7HXimcC$~Y;T^b zO{fV2v80+@P?;alL{FLVvUIaNQo7GL+v>8gL=OB>FYE?TC`OqdxkH0sR3q@BBN5Wy z;{1gdFkG~xK+w=N`g|_^)!9<^ca9+n7a+L3Fwn6jl@XnWx2ws)oJiUtC;iqIY zXf4LH^=_*4;`^l((HZW#VW+gRXD9ZRlupoMI&Tj&)pmvGyT$V1xsF;WU;Ph~r8DnJ z{`Dn&|Ag$a*BNvJ+HUMFSzM`FZf9I*Y>JVx2iHaQAWn&A2u*0>B|=|KeW|B;W#dz2pDzd}`NpJ_H+8Te*{{MThlxrw{A(5U*&!eL46DIQV-cOS995X?#dzy92ffDN@c( zFSgJR4Q*JVunhEx!j_VT^KxxAEg)#<8Yg%BHQx;N0`1#ADV!DH7mTCjpyATp^7Z(( znudO8;7IS&fvY0U?vt7*pWe7DiY-h0ZTlkGJe@xMisdW+EXi7|zsRv;W=Fwx%RaJB zHu)iERYN^zCI%pC5gmF8U>=+j&A75u2TWpVtGt>-Oj)s)`g{ZM8WpPa!KJ!T>I7< z=*A>T@mH``(n)#JlV!0+feu9Pm?w7<>jXjHmCVUsu-5jwB}DJbkXxC?t63w%_Reau zku@#^eFNRkp&ux))M%hk_dbPl!o74lxPU^<*!6y)9xK?~4AChN_}7<|i+#OW)A6rw z!o@4(2Ck}T!7R#usIW~DYTtwqT^u9x2|Um!gH3mMeU9A4Jf{1HZFuphEaqJqfq470 z^dPLjHxM*RR!MdC#QTz=Es|rz2jM8)2+HBjl8)C-F>QfH1F@r%IDFJrrvbL#XUTqU zqiePn;^aK}JSz~z<@8P&#AX)|zH%I0SJ+55Uzb0Zh+<2=5bs-}Bxl@D?FDUQ&_Yh% z8t0^6-yx<;CFqXZmz2v>Y>0%V(f5KR2}G`A15xXYWN>2$aH~sSb_Dt&16-`6eJ`vp z7n1=(3q~U-4R*`tx3$p5(U0Twjo&9%aC{97{Qwc4CI#&awUju%sMZ+Z$=uGHsL+d#jE*&O=$-A)^2)IBrh5clbO=5Mue9CtEUe-az`2~Y8v_@Gk zTRADxY`PY|}t4~3P3%b5DI>-d|72CN~B}s&lQ`0tf80X!(GHTs>lGTBC~O~IEuV2e3KZYS zkB*jIq;S)tk_VK+ObPHQsvAH9<>_j2Io-i;0OA>}PM#|*m=&Gf(fuS9L3F=?P|+rQ zHWh|vy1p-slLR{MBIH1Hy#MNHxtBRv(js2>L`noF(@*+8S0JwvY#0XI~GI!cDK8*s612Hm~nIEyWJ~aGTDe14TG$=;9pBO7&aOr!n6&)H}KkiDNZwP8SIQ#DjvYUMexLCFs zb(hI3PToP$PteI={>UJX4RGs*jF@=ISh`mK(2wI}5OlQrn9ZC|L&(Xf*+}-V?7@US zQT8o&Bb4T1AsS4SHf;WB8t9VkWH1MP-SA=kLqD*MPn3Lr3I+|L2v)?Ajds*>-K8>` zrOS>;nR{roIxc-Lv7al@0HSrF42WdVg=q2|xrP}gA>2PQI_3txI((~M<=6oKjLyk& zIh~Ha;fv1c%G~HROh(6LvXP9S8luc-%N2kx4!s*?kFLc!+iUo+4Z%83k*`@e7h+S; z5G;L;WUGfws~2L?vLPobs3e1R&;~xuag^;r8KW{}!|^wbv#oLz z#I0aqvbuGbTcqpV=Ju9v;U%awyj0Ku+F=!l5lgs{d zNtg=|73*XhRa7+uXh0mUBm4c$9oz8h2^mT9EcEcT{qJ-cNyJQ~;irv~zh+aIC^SyR z%P=Z{fbW~|pQEyiYbI4(`ZcJOHX}=nO=%1|{Tg5PE7-8n@(Rga z0oRY?^cz|%Ygrip?h8hoziE=)&+>! zS<;t`u?1FrAnvKA+Q)Y_7F%O{%hA9J_Je%ofOvAUtf;>|+yD)-HZ4CB$$Refdm&T$ zu;wqc;{^z|^*eE{4j12ZPji8cTsLiDTdKdb`f-qY==A4qQ>=~4v2yh1GeNA8wWQfj ze>SPRFY2}PA{hJrd5U6~urz=}H6*Q* zYZ2ehns5t^PaG0@kDt~od_N`0@6N*)(9>Z2eucAXK$0hLxC8Z;bOp^}|Hb%Srh-_FSM zO}{rN2!?XMuaY7Qt{Z9iPE@PQs^Ts}PN2>fng8o2nyr3x!!&uxKbyBGhm%!4A7_Q( z2_-YSpl^xfM}hhnhZAG$%-CGEm0R-^|2Xq zgAI*7@p8K#bz|k|SFDn6NH*G98kwtA>~B?BHG)py-f^X0^L?uYvnVsCxZAnOwVbYP z`e}d!OG|7&x9SIOiIpNYb^+I}M6l~}R2*qFY0!`31a8lfcOy5shC2b0XRV5fxAf7v;T9oyPLOHzBEhlbVqhJCczFP0zk}}pTHm&+=GgVSl z!YL|6-?fM~_t*^_G?y~sPNFp#N zaGsaG88^s~$?BpT zok(|(Vbjz_biy7r)`fkb-)DBAu%5&*%ETf?yJoC%c zRF%-qZvKG+#62adG3!&^PZNwT_s&oW^g@2vZYfVD$$}T6l2^!_4l8c)xb zcl?4<)*ftkymQFB=6xE^Zj%M|AhAo_lhEVNrlnlMD)r$Vk3 z$`}3f32t>8JgJ3)#z%#+*pE@90UB?TU5w8aUl$-sb7f(RP*_qLR*zF{vQ3bUZlqz| zWEJ<336SwS-G)@vxQC6Yu@IXls8-+J66M=ZSt$Pdc4Ht5vE_tnwd3DrWSa)nWrdeL z?kTgBWMm@Pp?zK!iCLR6vT1&2wY|7yY3v=RnnyZ6C_fI9SK_BeabYiKUyVIc zCj4c#ie+|SGSOWk*>P!vpVL}Ugu^aDhjV%Zn6Yq|v8KU930ik)mW|9pscS*qofs#-Qc za1>8vS~D6TN<;Tb)pC@H+|ekGqk812q_6F{DGfmMovvE?aj>=_BTps#kPw-J6v628 z1N*dM`a!TK;@$U%P%UJFrPI+6nx)!(oEfDY`wlO6jWad{1uj<~Rf!iijZ7Dc5X~2U zX%dApccsk7o`CBI6fOpTMI8w}Nw3kut0~^emi)|nQXwrrnQ4nj#N;wfCINO(l!bRmaeWQ=^5m87F zydNb8JBsB5=;~?965TvOo?{)PxmY)Rwpr%4xg<*c_vWd@`xYAYt)yGlNihe@;OhsQ zZYz``--JHVnR3uSzP1hPJE>+1tCH`b{-@QXoJ&__(BOeaStlvwL;`T1=%@RnJo9TU zhkk(gbhea#eQy{A1AR6{%84#TD2+mcdqOq1YpDq(hA0iBT`gxLQya)CN9x7-8=lqR ze*f5ZMqN13Rz$JmW!nnX_~EY&7fW>iFgf^1)M=zTTp^38Xu!YvL6?dh@-Fv*B61um zk%Q5h(lYZ3y$kZ|n*7ODjvdQX;$wj*&mF{bc7>yfqV0D_<%KgFqI&XIyHzVctA-Wq zr!(>bp&CA0hZpWXulmM6O)o4pV}IXD)zS~YEi**FKOx`QC5u)N1N|^XzO#wqG#%I; znxv9R6sL*WhSPDf?t_J4Z9(~NnrebQe72xe){%o$wMQuJC?F2)b8ld8ypRSOHA;Qe zCjW4l1{OP>tGiUx*#B{&YU*!=g960&CsZQK9z3TXX#BK7B@#%kxfr4jE7%wPh+V&- zgS^N~iyzu*pq!bZTHM(r;!3AZeK}Oyq)oJ4%3&|~5Jgj~ba5K(lr@A%Ktn$$_s={P zx7l7nwD$PtsER9z8kG(yhSyI1B#Q01bk#08g^CliM76y08ZEXJ%(;-faG?vsJvrz& ztukey|B5zTOv5LwB5rV6Y>`8p6KY9$LL|y7R&h_K=^sAP*di6nl{;;*1Ostl2bEaQ zTp{uyT9?bgNM;UbG@h+maigqCTPJar5Qbv}vs~IRkfZnMoMuTXu4~ozKBC9&Q0={m*-#VB0=j#bwQod}%p+?;j+KrY{E&S2U7C#Z`4h5j|tF zEG{rRJ(?2`9ZOUavBf@H8u+}ALiR}&zVXdBhLY^ls zC&G64Bq_SAmua98l(eHNnWTu?QD(fbjj!79n)gw#N5&Z%s&-ju=(kuU-O`L-Mmb-iImJmj!EUJ2|LcdZy{PRNnRuqCo=fD+&)2$ z6DD3-)e9foAuq6CMRi~OCs)Xe(FszV852~CuGfa;(4wDIEjU~P#W(VED`cU~DeKdi zTSrPG>r-v@v8u(7v%+o|?l?q;mG(UtW(&luB-M^PnrrA%g~ko@x~s;|F9>r1;@Pn( zzVW9~o|`vOwTxWH))1Xk0{f@hggi977AuF?5YhI)zli?Eba{(oQr&TR#yieSMJ&H= zi0We-Qe_d@#e91}gKd{X(KjHRjZ5TZ?lt-l`1Xx$s_m_PHA~+fe9p%C3z9OU%X{@z zLdvVU!)Ykw*pVoUzUdTJjs|WDzchDql<1rJ79)>yWV$$-F7r8T(=;H8U|+6O@t5_A zBFakoC41Y)=fdUlmsltwsTK@L$?UVhKa8aj8o#cSqD%YgY0Dq``}y)@10Vqtf`X&C zuYT~D*Gob5Q@ZBcFiel}v{%(%?b|8k-UzW%yExqmvyI{$kP8jh1$ zBUxVYZ$KbN$jM~OwTQSJ{W~f-gw6y1}V5z5Nb!5kR zxRXh-!S^mc8y+4A)}D#&_-Tz|mCJOx%QAc4!{?GUNtTnS1+jGlGzh<4HrAx1@=^L8 zS4kxQ_4_(RLzVL9?3Tt+5T;>liM$nAYomS+n=0LY2zwuhA$ck>zN;2nKWKYnp1j7! zT~tH5yw*Hbv(yg|S2a?NZ$2Hx<>iU;ZY1ak&z((nd)b&8tsrdgnj=g7s7~V-=dbB* z+HV^J{M*0KxbiX#Y z~TBsu1B(XlsXQaP&6SVo=mab8x}Y(xZgWtqH&8gYQ(xPj8HkOu++coGU%Z z?b-nxgQ0QA8PKqiYV_8FW{becF>q3$SM8=RP53?vhMa+%S8nZl(403IIR?rJS0CA& zf{Se!8t3MsEgjc-ZlJTIv@sKk7Twr%;EP+_b7bQyM0BJ6!29aB`COL+Ut1n7OD4Iu z5Mc(2PxOa#?*7|P8eJ@n4=Ms@NO=gjPjpjEX*;4!CV*mSWRckUA1cJa!9rxU%~jR> z%yEWBRM?WVi%o(~R%tca^zZ-Ls39y$c4EBzZhdRz$Znk{zq9RvP(H-vMJA#O zmUi4`6ZZNCcf4t^2Fjkg*gX&l)CoG-TrvB~KbwbB$jP2@(%p4yq%g0OU6AJPvIjCg z+qXu$l}8=pIip~->$pQgw7CHt`w`O0RV%s_nD z?p^2p$hkWB*0)Tmgw&D!z*b- zxsAi*_wBDl8R(CF@-5}nAZQIV=(;%fSMIG9gh%C|TjSlsWbxKL+2}IpuFdWtE<9)m z&_d3jh0EN-9Jio{wdtUXmZ(1KF4x1uYCCA|O7|dH@b-o~%%Fvn+|TKBS~+8}K?CAc zb56W94a;Tu1ounEi`Ldc42en#Oo1hnTA)eBL92Za~|^byS}&Wl{EE6S7aXxg%la$e9r5 z9%$6tIJ-=DsT?YiY~}C~#Vn1S&Ka_mlK`~UkN$kgb@!359}ZUioOjN-dl?=D;mIIp z=kfoqy|0a{s@VRgNKGx1La&O2Qdtz4g@tw_&z$Wk3>0OHg5o2FiK3z?8lrFz6*+)B z$djU^C@P8)S>D8CFS`~gX@;qkrIrtsY3aR|b+zAj*6hRKbN#>lzq@|iFPJm4)^|P3 z?AbGG));5}esvAO!M`3gPOI@p87k!NNaF}j#n+<=gtKWOz5R>}(3s@HI6@w)F)rc0 z0uVI|e50w|X?Ho|%EzV!HzkQLFv6g{N%V1dqX}c7B#Iik7V>-*-K)j&5Uf<#jQfFh zwCDq}U}vO_8Tbg+5dT83wat&$rAsLVh3K)yd7Q+M!nw&oDfrd_Ynwgo*^?P*Fz)wg zW8PU0tw9#gOV&2;s3a>)Gz3v5r+$5bJCFjlJKXqrD94_g(CZqF&&=!%++^sz85Dw9 zpJdBThW2WvkRfW8qi7&z;>#}gJ1wVaA*EH~ICc_&K|VHZsC%e!2tEpw%9KMNsxvP7 zyz3-7L>oH0S)4p|PGV3>9(ppsxQL4s9B2mCx;FG{U*ig{@~U@=Kg|7_aSAc4QtPbK zVX)kV!Bj=5C=us_>8#*RZG~kv@5R`1uB=NjxKnes&1COrhBO7T;=4Y%z)X=ri zxv}DB6)0@t*nt&+FMe=w8Sa@RrnF9HIs7G`{n;yza7>LwN;`ZCW^7&6G6M@YJQ}x$ zWnOdI<^NTqC{Z*F2$P_cDTnWk6`%Z}4j2NyO&gAt-cPX5Eo(5+;h)8bPjD2$G!$z5 z)D1Mg=K}bLu8o+vhrCs)bl`Fe(}w@zCK}5Bl7h(xO@0j$4FQ~5PZ1q4tjTEjEs=XR zTYtpDXyeCaR!bc;rV2_AQU}anFTg)oxDn;e#t?M_yMjJqS&aBj#lx5dyWFf@aBn2+ zA|Fi)>su!NdO}@!$w0lP#r(+ikuJDhCwu6v9Z5q*s6il~0Co6*xu zMX8#M&Zj3{ZQ{ss2BzQdZS+9ThNMb3y^FWujp+t{Rh}G%ypr936oQ#GLOM~qj@?`X zn;z3dJ&#;rX<6&~bJ2#kO6)9~*fbsK=i3bFA^$Leb;v<(eRYH=gng`IxwXyT=eq2o z$9ouk-)Qy4r^nqC4cOw8G!#+1flfa52jhTnHf^MPH)>;jjV~Bakv=x+6+0EZex0R- zPIN82gAcWTfb*X!#|`grklJ@y$Kx2|7-PA}9#A53&4C5XeY z3ka%?9DaYiNJe-RG$w)R+DIH$IB5n@G0|{Jpl46c=d4Zj5?HuVO3&Wt$d=<8qulF^ z+em=nfGHL9=Yq)R#gVKJHq@v=g8XsSf^;dCWmL~b;}&{0?4o>ZR6>%0%ulX(hf$;Y zi7&tQ%9tAajy=uL4IF%ytfX>4?Kx%dwj z0H6!yAEUFcicAEhfu&i1lvKxhuVSJs{^)&}<@DSuf0d7>js7`aoWq**VMSv&3f#rH z*2@eLJ(EQGBYU_8IO$r%gE`{ot_ByNnWjYy^b;$F*(DCi+NMQ}!@)lEeXx|ZP=JF< z>8_~)3Z`8ahj2s$)s?i7y@RN0_&TS?P-964aUH$VMKtfA_#8*N9pJ*UM4U^ol%ktL zK}y<)okE;ink9cVYaH_t%P{>i?E<1WLi8n8zd+P%f_9AEV$Spmcx@JFqS29 zWFfWr{4#IoZkFy4frIqFnZZw>qic~bxYHvqJSTyv8CwJJ$?{18pgph-QrFw)3=0pjBO^-PpC78#Z!Qs*vmxzwFaZeFGVkM(lRP$I=*YnSGI ztYn2l?b8+TOLT997aE$SL+^K^yzYT*4d4(RoVbaSl7mk*e$Y<_{v3eW|=XqwJFEolP7hdAJ4OIiXv)9a*xI$VVCz`|z{wiQYjiGj;3F{yD)s*F5gGGvY zjNq_@J~qLZ9zwhW{!mUg=7wD?RLf9-DAcvF{@p~LIsb@}9yAvHCC}VQRW#NX8t)*X z(0!@{p+@*kk&9H=_*Ef|>=tG%`zPOMpbEzN_*nbw`v><1MjF-8pPtitGYtp`8VgHB zGG-a4Pg(0e-t@d zXbje(5Ye5ELAu^Ko{qbEp7P3`lvgb%Geq@XEY6Q@>tqWroE|F*%p3rUXkxV}z#V1y z!}QZVUZTL9DxlD&MSboe3SLrM;HsdaleG)JUL`O*nl`ShCqnM#xM+;SA(BLRp((Ad zje8(b)GcW5(nr)EKGrsv2c-ZyYTG>cmPJ#dagIXl}U0eGXVabX$V8 z&6-fBaG-JdlE}v`BB-wXV;nYxZzHznu&V&K{$UMP*1%)a#v@7fZAAXTKa}OhFT5;H zmp<)mf*Odo{2Jm*4`S%!Kki`d@@GpbeO;SSd_&lA1G`iZYS^?1o8rYj3`4uiKo5A z;nrF3qQ9-CqW9}9U@KYc+T>t=>f8R`Tu>_q=}F7V#9{6}Mt7HLsqOSAaE zJp9GLCIhLjy7^p*KPkG-I(V*nohUAAV#&{3sDc-P#xHT=C?rjBia)uhuh@&@KhU^j zt7(%528j=_Ac6vAsL99#{taT#&_r>Wj6IxvIF?n90%%^>qOp+K^|jyFE@-A}Q^FFA z-0r2U(+jf38_+P<@I;kR@dH4ilwZluJfCa zp~s|uW4))iuM?@R-C3|K#(GcTLdiuLYU-dUF&e8q5<1hEKHg3nE$_L$v!{L-EH2=V zL->Z$`qZOt;sO>!WdWMlw5g{Siwg+O2T}Rhv^PVmt?olBWp(92u`y+q{YZF;HEnsn zkFHJI=Pv57LITm{v=#+OJ+K#HCz1v$DDHsC99$*=1EOthosa0=vRe;pKeV^RH zucQ=d%GhE3tj{1~25v1iuWQrq^%7^z2^z~X9Y@U0jqbqxw!0K=dW4%edpw*Al!D~- zS-3$+(qJyr7akR7v9t$*$_uAwxr+~Des>8seF<(J#TF+xDCk8pooAX7Ts9pL z`^@CwitVlp@nM8=DX`VFnIRKtHIg)ehRGJGDLzC3E=fb-I3mP3EJUEaGUbfB7K?K# zH@4K8uFZ&TC(i7COKMDFK|xWJ_z(#W09_T#zCWCn9yr1NQ7;uV`@uk3y2)Gi21R4$ zt#Au2`{9qG!;x3*U0uX{=_O=J%nSV32iEk_q1BJ3m41-j`B(&vpkDllI^@7a1IAk zS~O8QogZ0h9N1fDy$>}|D@>ao?QiVOye3gvaX(L(mY5Z8eDdRUT;b^^9~7)=79Sz- z034*mHZ2k7_&&nke28;TJn0|1sq_!E{DVSWTOdlsd**Q$6xg%{(~epnZddIx<$@WP ztUVDN=ca&Z+JbrR)`zjx+TA(DUpOby+IikvJO-gq*A^~}wswB~h($JG91E8;SReSa z&edPuT2Wzru&z-4suZ+%UO1Kf$CqsjAJMfWoLA4r_LIq0*B1416Q%JSX|Eh+(R+4r zAC_HAL!kze5x$F67PzJI!bLc?mua4ig-!-*$*=6ObjQX}Ti7;54k*bPe)a#tFb>fr z(c#)+iyQU+qTKqfatGAFr^a#f!wOh*Ep?zbt$8rgg7y{*7tC`=oytE{CtNbjZf)C1 zE$g$AmwXT@-bW-8Fetm&aRFO-uQ*rFwx{o=^599VAQnF*H{R+yHq<#i*EM^7H_AVg zCkIERvD8slt$wlUv@r{3Uz9|%-G=a<)+uQQI|WjC^a&n=7)R<`Rn|8W`C~*WD0N|w z^$mPBb$9k#`)58h6W{8nuDIC0tE0I~2lE^kwRCOiV*ylfpf?kBNDo>XkWPz#dfD|# zvJY^x7QK_~>gwrdgQx%@5pu50O2GAK1@hjXoOh?agmol_K%0rh;%1Bs59dMYjI6P8Q3d4$oaY*Vj)HUtne^1&~KAN_&LnKXFsO}_ZX;&81Q2K8=AGU_&bZzB#`IHlFV`ym#SzsJ|m$I zrR2;$fi(HbIM+Kn^SOAMwr)1Rl31siL;YylttTzANxf81=CDYL^_lJ*wrpDF@EV%x zH__$%naMXP8HZRQn91e%Vsf$a1angKa8Qbyz?z? z&LJoBb~{?u{4{d`7T{=ZqU4yJ44pNW`Q<@M!j~sds!=Ml(LrzA7|GJIp)&udqNHO7 z<*!a)FVWPzWNCS&)6Bp6&|A^j?1fC!(J6-#v7HYSs3s1td`c|c$4!{1W8z?1VZKcv zkFGfa1TFbAkb$X!zn`EbNYZo)G-~C?h~wFEO2qm)+N+N22#lsh$Oa6JNka|S9C`;z zJ$KU8Zyo!>DHqX$QXhCw(On&bOUl#K6v{Hk2r@k1pg>kr8Orgm2x*KScVLab2Tl=en2C9l7W z{R1il^xqO`5k4cYLpYWttIc9cUZF1h05Dz4y03;7VTA-5Qv+M(ZBJ`2FBOZ=II^ZE zQl{yaZ0xKy4w{O|glq!6f<~uyG^MVMltm%UYIB)J7V^#3O6ysc0GfX$j(tqg7#={2 z?jP^!9$ByWP@H=$D@XxL@S-Gqnxnb`mNm|u#$nyE);TlCivNbDJ6Y-ZF+n7NVf&JWV!Jb%e+o*>`(V#<+MkwrXVp&v;_D3lLgF(I1 zgR(vRDD^AVU;LS9FXYT{N301LrAnKzp9{pq5R`#&`AtOui*Bg>-x*EkOvcEITW!qgZ;OvArN{yV?s$h+v1+TP9K%QOp5(qiMg1z+PY*NOT@xv$caUo`k&PRrqcMJgx7v4Z;DPs>Z8(Xd+g7@Q31(BODZ!IZ5hN=_6-~ z8^wJyTVBRs7F7CEG7>9phL5pp=ODX!6lO{nP{97NDk6+l;VfW_PFG=LdC5h!w+7a_ zwyIqsEx|?SZcb@eJ&{fe#_Z$qG(}EGR&AL`x#kHfxD}dojG`&<(^~ktqVcq#xw9YS znIq_+#tW6S0Fx_+QrZO|mCoF*P`+2+u-+^$Wu zYVHs0K>&vuKYP%e{BmdauxYD)N~gG&3)xSV#?HpmM6V)NVT#mPZpQ|SM<|a(stRUb zqBIBxT_{;{$9YgR@}HXGqe*QyNDS-7uYNfank{&$F4PKbXuu#!c=lh`2;c%kvl* zSV1g)-jmG~g8+yY9C`Diu`;1aeW*T{pXN_%9uASvRYBgd97=g4*gsW%YS#a8$2B6PDFy-3$ z9b+l5b3iLD{VQnr(5~F04$BbD{1>Z8pYS>>2*%R2{J;Yg`rEUzK#iyQV{+)tOFLY$ z(pf~CCE?U%_!2e)8*253J`|)f zwVKdBcc-x2%T5IW*r$Os#0)!OqN`88L?duM8`@Itu=>;i8sXKJai9=fE_R?1-T5plu=o-GXX0#k(*0rL`jg<8ue~BxXDw-8aBUAF&>DZLTT@xr9 zFD+>P^53rE(#0~Ovkn6i3tVrPj1z_1{a zTFfhXCa|*0G`f1TG?aYMfSe|QQSV9!eD)AD1d|bMd_4<%z_Y1DX6%QCq+G0W5KVo7HT{dOd}8q2rjC>6wj!j z*#4K;P>i&AhlA2TZk5`m_(}}r^qwK->&leHAGV|P-{-p&RGe5&;SJkag~|*|5-AKi zmB3&*u>EHxb$^*Z0j1_qaJ!vx6gV!R27Ckshte&L2%E2UnxW+NgESV01n{d8t|UC1 z#y%e@+big6rY0I!j()-uMdniSuSVlWg;ufd$!_I<3W?5br;gS zGV1(vI_m>8Y+C7?VU%}~uNF|Ul+Ha*xyN{eTPdg%ADS_yZJF&I8Y^i^qBh9#K2Qyx zx>ovpFpcSQ(Piv4k491P1*=QzrMMu8Fd?>IDpW#!$(GnXOd#-~O@Y_d>Eib)=Qk!3@#(7K4!#+qI)S zDF-=aWdVe?K;Ql<5^sU9@twVvE7 zp?TCO2%{-iJXsbt!`cOL#uc24b^r%P2wf}dT}gSts=p{h{ZVCo4tL>wq88kt)8TGb z-ye=zd#P_ETIY$YIq^{hq^5#?#UX)$B+=Qqs<3WtBjvxwvG}S< z+0hDPS>`qVSXFF4FBK)2p_obrurn2+Bu$+q<{Im|xEnTP1HKMF1rF5sGf}*ZFkm!M z#$H!hP06iyn%85$vY`J*lBJ0>*TcAi0}mLG(2RPZZx7(ekv<36o*BR;C zSRczBu~_Ot5wf=USqn1ao6%W`-L2rNLMGuy(W*n#F z5d8M|1o3h}y(Fk~y20ll6r^*%1t(qG@Nq0N9TnBJeC9-T!VQtTDWx{T z0yz|y4cq$Bk_|rmzGdhe&NGWWIxD8QaCm?`ceMvJ!}TB*0qgsi>V`0 zFr|P79-Fq|+gR(;f#>9+oqTNC#+AX;@yI3?zh2s<;^{=nxOj|}>?R8~HdY$H;m~GV zST032sR)XrWm}h9@T*c`1@?jB5dU!s&JX*7M(iF+YojsI5ZN0l(qE!iv9|RPX5(mv z0-L*-jdNN+sf4Qt2@rEI$AvB=Yt-;-v<7;3h+fW7ssb}dJ6^yg&R$vZZLIMXVrpc8 zBKozjWwz_Qn5JzS6-h&%9l*13;25H9%4wjbFSXpxfP$%Y)DG!S?gLv!x@l$%bws8T z$e}!WQ)3VMt*rBvf0#9j>Zse!23N1#T(X!l{Be z!Qy3PCj=Lzpv}3vjrWn1S=L~-mG?B#Llghv8Jm(NCdTSzrWi&Io3^dVm7>e0e4k@LXt-mZPhu)-_Iak<} zK&xw&6O(8bf=WgREWrIf0+4po{6&E?rc<3nj~M{#H-(;xiOLmK#3SE-$uZ zCDydcW$nm*WPs$NI&$S(ku*DV8!L!aSoz`}W1%_QQW}eQP*5F$sZ>(&<}O2>^CsIItd z8{#m2nRh@si$c2X*=sbno%+y(_-OL?2J-h9!&?-9kAnUgmM3v2PXbeHpX?;|B5oXi zm<7*xh`mTs4W!D0wk`Lkl&&4+a)-$lOM7a_Ds4`GOuezs(GCmu~UZ{}b{;2Bt z7o{!Mj4Z4uv@xGqE##Q)3PtiSnG$SlB;Gniap4_59LTYHm$1R zGCgd4jk^c9prA3{*nq{*ddRX_YOLzVtJE%QJo_o*sQT$VwS(h=gJQe=H9;$T@DWu7 zYt=gE66@srQ=s@X9iN&WBs|tDQT$fNZMUA zCc))F+ryFE1ZnYFe0}?V*!GP#rN-og94jl5jJt7k7-le4cs80=PFHyq6c$vz; z_5B_DzM&NeDuFK4D79l-(;#7fN2N#)+VNXCt>_oPE~RL|aqog#I#hoN)2e%h8GEtH zBDKyns$cLH51sx-x`KQ(t$Luh7=sOepj6$X8aL86^!iJpRBWr$E9t3GhWxNnYxLE# z(#0oOM1U+xW7UbF;*vWk0d`WUCvttMgER z&;+PY`DyjHQIy$>_Xy1P`s9HRVd_7M@*z4v1a|6n9aM%W^xr#~U>W@&z#Y*tDJA9Ym=^ zT^ztTc3P6i_ovTX(~X^vWr$Lk(rlt>JAILfqH8~=_@MD}qu9FsYo~%xFe_ASMMz?b zvp4Kat2P2L=Wo&J&Qeb?!OWG#(06q#r{y>q+uixn+WBdfk%RY3i?Mfgjzf(%orco2 zo!7Ar^Y#q(G5CnC?P?!Q%PXd_!+^C-+tu|JJs!SJj>&Rzgc{EuG~{ir_*FUXuE=m> zG=NF1qYyQ=)KGuiCoXZ+GmT0?{kyA!O6+vI_D4~FEHIyPBHgvgPi#G}$pYnLyLJSL zt!6Iy5^39ApJ7RD>M_v?+fVM&F1x-uPxeob@Svn>d=Xopi^Gx^oKCmvrXMZ)^tx++ zt!aN#Y;he_(zTinXT_G*$z5yij}}{TDh((D;S#9PIY4anZ8=e9(`u|uv;vz;kW~3t z&9Fdm;@otX2L0&v-gTkg!`9cTtF zwdvwxWU-PofW@Z0*WP*4%j(`c zh{Z=O6wKIP-xZsF@DJq<@5~rn{}A5$5(ij==z>nh+1{&o2zIb6iQSB|eN-fq5-!2F z-cz0PG-bIOWsJhoFHowX@15)A#<|P;n2QDh?XLU!+1b$UD zGXHko>h8x0mQ=wX$kOcIi?4TOSzgIF8emz4>S)iy)+;L$I$2~K8@Q+%+Y&liJE)sj zrK0sgZBBn9mg0j(;((He40>|JWs#1x3#gB6JLpM#ai+gm#{ISyzy>{??k$#nX6JDX zX!Sw;4>!_MTnd3dSo}f3I4!sYq18~I;u3X#CE2Icv``): input runs live under -``/P/output`` and outputs are named -``validation_psf_conv--.fits``. ``v2.0`` removes the patch concept: -input runs live under a single ``/output`` root and outputs drop the patch -token (``validation_psf_conv-.fits``). - -HSM ellipticities and sizes are no longer rotated here. PSFEx and the in-repo -MCCD interpolation now measure adaptive moments directly in world coordinates -(galsim ``FindAdaptiveMom(use_sky_coords=True)``), so the WCS-Jacobian shape -rotation this script used to perform is redundant and has been removed. - -Caveat: the MCCD ``PSF_MOM_LIST``/``STAR_MOM_LIST`` columns are produced by the -external ``mccd`` fit-validation code (``mccd.auxiliary_fun.mccd_validation``), -which still measures HSM moments in the pixel frame. Those shapes are therefore -still rotated into world coordinates here, via the WCS-Jacobian rotation, until -``mccd`` itself adopts ``use_sky_coords``. This is the one branch that keeps the -rotation; the in-repo PSFEx and MCCD-interpolation paths measure adaptive -moments directly in world coordinates upstream and pass them straight through. -""" - -import sys -import os -import re -import glob -from tqdm import tqdm -from joblib import Parallel, delayed -import gc - -import numpy as np -from astropy.io import fits -import galsim - -from cs_util import args as cs_args -from cs_util import logging - - -def collate_paths(input_base_dir, output_base_dir, patch): - """Collate Paths. - - Return the ``(input run dir, output dir)`` for a patch. ``patch`` is None - for the patch-less v2.0 layout, which drops the ``P`` token; v1.x - passes the patch number. - - Parameters - ---------- - input_base_dir : str - input base directory - output_base_dir : str - output base directory - patch : str or None - patch number, or None for the patch-less v2.0 layout - - Returns - ------- - tuple - input run directory and output directory - - """ - if patch is None: - return f"{input_base_dir}/output/", output_base_dir - return f"{input_base_dir}/P{patch}/output/", f"{output_base_dir}/P{patch}" - - -def output_filename(file_pattern, patch, idx): - """Output Filename. - - Build the collated catalogue filename. ``patch`` is None for the patch-less - v2.0 layout, which drops the patch token. - - Parameters - ---------- - file_pattern : str - input file pattern (e.g. ``validation_psf``) - patch : str or None - patch number, or None for the patch-less v2.0 layout - idx : int - exposure run index - - Returns - ------- - str - output catalogue file name - - """ - patch_token = "" if patch is None else f"{patch}-" - return f"{file_pattern}_conv-{patch_token}{idx}.fits" - - -def transform_shape(mom_list, jac): - """Transform Shape. - - Transform shape (ellipticity and size) using a Jacobian. - - Parameters - ---------- - mom_list : list - input moment measurements; each list element contains - first and second ellipticity component and size - jac : galsim.JacobianWCS - Jacobian transformation matrix information - - Returns - ------- - list - transformed shape parameters, which are - first and second ellipticity component and size - - """ - scale, shear, theta, flip = jac.getDecomposition() - - sig_tmp = mom_list[2] * scale - shape = galsim.Shear(g1=mom_list[0], g2=mom_list[1]) - if flip: - # The following output is not observed - print("FLIP!") - shape = galsim.Shear(g1=-shape.g1, g2=shape.g2) - shape = galsim.Shear(g=shape.g, beta=shape.beta + theta) - shape = shear + shape - - return shape.g1, shape.g2, sig_tmp - - -class Loc2Glob(object): - r"""Change from local to global coordinates. - - Class to pass from local coordinates to global coordinates under - CFIS (CFHT) MegaCam instrument. The geometrical informcation of the - instrument is encoded in this function. - - Parameters - ---------- - x_gap : int - Gap between the CCDs along the horizontal direction; - default is ``70`` (MegaCam value) - y_gap : int - Gap between the CCDs along the vertical direction; - Default is ``425`` (MegaCam value) - x_npix : int - Number of pixels per CCD along the horizontal direction; - default is ``2048`` (MegaCam value) - y_npix : int - Number of pixels per CCD along the vertical direction; - default to ``4612`` (MegaCam value) - ccd_tot : int - Total number of CCDs; - default to ``40`` (MegaCam value) - - Notes - ----- - This is the geometry of MegaCam. Watch out with the conventions ba,ab that means where - is the local coordinate system origin for each CCD. - For more info check out MegaCam's instrument webpage. - - Examples - -------- - 'COMMENT (North on top, East to the left)', - 'COMMENT --------------------------', - 'COMMENT ba ba ba ba ba ba ba ba ba', - 'COMMENT 00 01 02 03 04 05 06 07 08', - 'COMMENT --------------------------------', - 'COMMENT ba ba ba ba ba ba ba ba ba ba ba', - 'COMMENT 36 09 10 11 12 13 14 15 16 17 37', - 'COMMENT --------------*-----------------', - 'COMMENT 38 18 19 20 21 22 23 24 25 26 39', - 'COMMENT ab ab ab ab ab ab ab ab ab ab ab', - 'COMMENT --------------------------------', - 'COMMENT 27 28 29 30 31 32 33 34 35', - 'COMMENT ab ab ab ab ab ab ab ab ab', - 'COMMENT __________________________' - """ - - def __init__( - self, x_gap=70, y_gap=425, x_npix=2048, y_npix=4612, ccd_tot=40 - ): - r"""Initialize with instrument geometry.""" - self.x_gap = x_gap - self.y_gap = y_gap - self.x_npix = x_npix - self.y_npix = y_npix - self.ccd_tot = ccd_tot - - def loc2glob_img_coord(self, ccd_n, x_coor, y_coor): - """loc2glob Img Coord. - - Go from the local to the global img (pixel) coordinate system. - - Global system with (0,0) in the intersection of ccds [12,13,21,22]. - - Parameters - ---------- - ccd_n: int - CCD number of the considered positions - x_coor: float - Local coordinate system hotizontal value - y_coor: float - Local coordinate system vertical value - - Returns - ------- - glob_x_coor: float - Horizontal position in global coordinate system - glob_y_coor: float - Vertical position in global coordinate system - - """ - # Flip axes - x_coor, y_coor = self.flip_coord(ccd_n, x_coor, y_coor) - - # Calculate the shift - x_shift, y_shift = self.shift_coord(ccd_n) - - # Return new coordinates - return x_coor + x_shift, y_coor + y_shift - - def flip_coord(self, ccd_n, x_coor, y_coor): - r"""Change of coordinate convention. - - So that all of them are coherent on the global coordinate system. - So that the origin is on the south-west corner. - Positive: South to North ; West to East. - """ - if ccd_n < 18 or ccd_n in [36, 37]: - x_coor = self.x_npix - x_coor + 1 - y_coor = self.y_npix - y_coor + 1 - else: - pass - - return x_coor, y_coor - - def x_coord_range(self): - r"""Return range of the x coordinate.""" - max_x = self.x_npix * 6 + self.x_gap * 5 - min_x = self.x_npix * (-5) + self.x_gap * (-5) - return min_x, max_x - - def y_coord_range(self): - r"""Return range of the y coordinate.""" - max_y = self.y_npix * 2 + self.y_gap * 1 - min_y = self.y_npix * (-2) + self.y_gap * (-2) - return min_y, max_y - - def shift_coord(self, ccd_n): - r"""Provide the shifting. - - It is needed to go from the local coordinate - system origin to the global coordinate system origin. - """ - if ccd_n < 9: - # first row - x_shift = (ccd_n - 4) * (self.x_gap + self.x_npix) - y_shift = self.y_gap + self.y_npix - return x_shift, y_shift - - elif ccd_n < 18: - # second row, non-ears - x_shift = (ccd_n - 13) * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 27: - # third row non-ears - x_shift = (ccd_n - 22) * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 36: - # fourth row - x_shift = (ccd_n - 31) * (self.x_gap + self.x_npix) - y_shift = -2.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 37: - # ccd= 36 ears, second row - x_shift = (-5.0) * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 38: - # ccd= 37 ears, second row - x_shift = 5.0 * (self.x_gap + self.x_npix) - y_shift = 0.0 - return x_shift, y_shift - - elif ccd_n < 39: - # ccd= 38 ears, third row - x_shift = (-5.0) * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - elif ccd_n < 40: - # ccd= 39 ears, third row - x_shift = 5.0 * (self.x_gap + self.x_npix) - y_shift = -1.0 * (self.y_gap + self.y_npix) - return x_shift, y_shift - - -class Glob2CCD(object): - r"""Get the CCD ID number from the global coordinate position. - - The Loc2Glob() object as input is the one that defines the instrument's - geometry. - - Parameters - ---------- - loc2glob: Loc2Glob object - Object with the desired focal plane geometry. - with_gaps: bool - If add the gaps to the CCD area. - """ - - def __init__(self, loc2glob, with_gaps=True): - # Save loc2glob object - self.loc2glob = loc2glob - self.with_gaps = with_gaps - self.ccd_list = np.arange(self.loc2glob.ccd_tot) - # Init edges defininf the CCDs - self.edge_x_list, self.edge_y_list = self.build_all_edges() - - def build_all_edges(self): - """Build the edges for all the CCDs in the focal plane.""" - edge_xy_list = [] - for idx in (0, 1): - edge_list = np.array( - [self.build_edge(ccd_n)[idx] for ccd_n in self.ccd_list] - ) - edge_xy_list.append(edge_list) - - return edge_xy_list - - def build_edge(self, ccd_n): - """Build the edges of the `ccd_n` in global coordinates.""" - if self.with_gaps: - corners = np.array( - [ - [-self.loc2glob.x_gap / 2, -self.loc2glob.y_gap / 2], - [ - self.loc2glob.x_npix + self.loc2glob.x_gap / 2, - -self.loc2glob.y_gap / 2, - ], - [ - -self.loc2glob.x_gap / 2, - self.loc2glob.y_npix + self.loc2glob.y_gap / 2, - ], - [ - self.loc2glob.x_npix + self.loc2glob.x_gap / 2, - self.loc2glob.y_npix + self.loc2glob.y_gap / 2, - ], - ] - ) - else: - corners = np.array( - [ - [0, 0], - [self.loc2glob.x_npix, 0], - [0, self.loc2glob.y_npix], - [self.loc2glob.x_npix, self.loc2glob.y_npix], - ] - ) - - glob_corners = np.array( - [ - self.loc2glob.loc2glob_img_coord(ccd_n, pos[0], pos[1]) - for pos in corners - ] - ) - - edge_xy = [] - for idx in (0, 1): - edge = np.array( - [np.min(glob_corners[:, idx]), np.max(glob_corners[:, idx])] - ) - edge_xy.append(edge) - - return edge_xy - - def is_inside(self, x, y, edge_x, edge_y): - """Is the position inside the edges. - - Return True if the position is within the rectangle - defined by the edges. - - Parameters - ---------- - x: float - Horizontal position in global coordinate system. - y: float - Vertical position in global coordinate system. - edge_x: np.ndarray - Edge defined as `np.array([min_x, max_x])`. - edge_y: np.ndarray - Edge defined as `np.array([min_y, max_y])`. - """ - if ( - (x > edge_x[0]) - and (x < edge_x[1]) - and (y > edge_y[0]) - and (y < edge_y[1]) - ): - return True - else: - return False - - def get_ccd_n(self, x, y): - """Returns the CCD number from the position `(x, y)`. - - Returns `None` if the position is not found. - """ - bool_list = np.array( - [ - self.is_inside(x, y, edge_x, edge_y) - for edge_x, edge_y in zip(self.edge_x_list, self.edge_y_list) - ] - ) - - try: - return self.ccd_list[bool_list][0] - except Exception: - return None - - -class Convert(object): - - def __init__(self): - - self.params_default() - - def set_params_from_command_line(self, args): - """Set Params From Command line. - - Only use when calling using python from command line. - Does not work from ipython or jupyter. - - """ - # Read command line options - options = cs_args.parse_options( - self._params, - self._short_options, - self._types, - self._help_strings, - ) - self._params = options - - # Save calling command - logging.log_command(args) - - def params_default(self): - - self._params = { - "input_base_dir": ".", - "output_base_dir": ".", - "version_cat": "v2.0", - "mode": "merge", - "patches": "", - "psf": "psfex", - "file_pattern_psfint": "validation_psf", - } - - self._short_options = { - "input_base_dir": "-i", - "version_cat": "-V", - "mode": "-m", - "psf": "-p", - "patches": "-P", - } - - self._types = {} - - self._help_strings = { - "input_base_dir": ( - "input base dir; for v1.x runs are expected in" - + " /P/output, for v2.0 (patch-less) in" - + " /output; default is {}" - ), - "version_cat": ( - "catalogue major version, allowed are v1.3, v1.4, v1.5, v1.6," - + " v2.0; v2.0 is patch-less; default is {}" - ), - "mode": ( - "run mode, allowed are 'merge', 'test'; default is" + " '{}'" - ), - "psf": "PSF model, allowed are 'psfex' and 'mccd'; default is {}", - "patches": "(list of) input patches; ignored for v2.0", - } - - # Output column names with types - self._dt = [ - ("X", float), - ("Y", float), - ("RA", float), - ("DEC", float), - ("E1_PSF_HSM", float), - ("E2_PSF_HSM", float), - ("SIGMA_PSF_HSM", float), - ("FLAG_PSF_HSM", float), - ("E1_STAR_HSM", float), - ("E2_STAR_HSM", float), - ("SIGMA_STAR_HSM", float), - ("FLAG_STAR_HSM", float), - ("CCD_NB", int), - ] - - # Extra columns for MCCD:737 - self._dt_mccd = self._dt.copy() - self._dt_mccd.append(("GLOB_X", float)) - self._dt_mccd.append(("GLOB_Y", float)) - - def update_params(self): - """Update Params. - - Update parameters. - - """ - if self._params["psf"] == "psfex": - #self._params["sub_dir_pattern"] = "run_sp_exp_202" - self._params["sub_dir_pattern"] = "run_sp_combined_psf" - self._params["sub_dir_psfint"] = "psfex_interp_runner" - elif self._params["psf"] == "mccd": - self._params["sub_dir_pattern"] = "run_sp_exp_SxSePsf_202" - self._params["sub_dir_psfint"] = "mccd_fit_val_runner" - self._params["sub_dir_setools"] = "setools_runner/output/mask" - else: - raise ValueError(f"Invalid PSF model {self._params['psf']}") - self._params["sub_dir_psfint"] = ( - f"{self._params['sub_dir_psfint']}/output" - ) - - def run(self): - """Run. - - Main processing function. - - """ - # Guard against a mistyped version silently falling through to the - # v1.x patch loop (e.g. ``-V v2`` or ``-V 2.0``). - allowed_versions = ("v1.3", "v1.4", "v1.5", "v1.6", "v2.0") - if self._params["version_cat"] not in allowed_versions: - raise ValueError( - f"Invalid version {self._params['version_cat']}; allowed are" - + f" {', '.join(allowed_versions)}" - ) - - # v2.0 removes the patch concept: a single patch-less run root. For - # v1.x, iterate over the requested sky patches as before. ``patch`` is - # None in the patch-less case, which drops the patch token from the - # input path and the output filename. - if self._params["version_cat"] == "v2.0": - patch_nums = [None] - elif self._params["mode"] == "test": - patch_nums = ["3", "4"] - else: - patch_nums = cs_args.my_string_split(self._params["patches"]) - - do_parallel = True - - # Loop over patches - for patch in patch_nums: - - patch_dir, output_dir = collate_paths( - self._params["input_base_dir"], - self._params["output_base_dir"], - patch, - ) - print("Running patch-less (v2.0)" if patch is None else f"Running patch: {patch}") - - if not os.path.isdir(output_dir): - os.makedirs(output_dir, exist_ok=True) - - subdirs = f"{patch_dir}/{self._params['sub_dir_pattern']}*" - exp_run_dirs = glob.glob(subdirs) - n_exp_runs = len(exp_run_dirs) - print( - f"Found {n_exp_runs} input single-exposure run(s) for patch" - + f" {patch_dir} ({subdirs})" - ) - - if self._params["mode"] == "test": - exp_run_dirs = exp_run_dirs[:2] - n_exp_runs = len(exp_run_dirs) - print( - f"test mode: only using {n_exp_runs} input single-exposure" - + f" runs" - ) - - # Loop over exposure runs - if not do_parallel: - for idx_exp, exp_run_dir in tqdm( - enumerate(exp_run_dirs), - total=n_exp_runs, - disable=self._params["verbose"], - ): - self.transform_exposures( - output_dir, patch, idx_exp, exp_run_dir - ) - else: - res = Parallel(n_jobs=-1, backend="loky")( - delayed(self.transform_exposures)( - output_dir, patch, idx_exp, exp_run_dir - ) - for idx_exp, exp_run_dir in tqdm( - enumerate(exp_run_dirs), - total=n_exp_runs, - disable=self._params["verbose"], - ) - ) - - def transform_exposures(self, output_dir, patch, idx, exp_run_dir): - """Transform exposures. - - Transform shapes for exposure for a given run (input exp run dir). - - """ - output_path = ( - f"{output_dir}/" - + output_filename( - self._params["file_pattern_psfint"], patch, idx - ) - ) - if os.path.exists(output_path): - print(f"Skipping transform_exposures, file {output_path} exists") - return - - psf_dir = f"{exp_run_dir}/{self._params['sub_dir_psfint']}" - try: - all_files = os.listdir(psf_dir) - if self._params["verbose"]: - print(f"Found {len(all_files)} file(s) in {psf_dir}") - except Exception: - if self._params["verbose"]: - print(f"Found zero PSFEx files in {psf_dir}, skipping") - return - - cat_list = [] - for file_name in all_files: - if self._params["file_pattern_psfint"] not in file_name: - continue - - tmp = re.findall(r"\d+", file_name) - - if self._params["psf"] == "psfex": - exp_name, ccd_id = int(tmp[0]), int(tmp[1]) - elif self._params["psf"] == "mccd": - exp_name = int(tmp[0]) - ccd_id = -1 - - if self._params["verbose"]: - print("Match found ", exp_name, ccd_id) - - psf_file_path = f"{psf_dir}/{file_name}" - - try: - if self._params["psf"] == "psfex": - psf_file_hdus = fits.open(psf_file_path, memmap=False) - psf_file = psf_file_hdus[2].data - psf_file_hdus.close() - mod = "RA" - else: - psf_file = fits.getdata(psf_file_path, 1, memmap=True) - mod = "RA_LIST" - except Exception: - continue - - if self._params["psf"] == "psfex": - # HSM ellipticities and sizes are measured directly in world - # coordinates upstream (FindAdaptiveMom use_sky_coords=True), so - # they are passed straight through; only positions are collated. - exp_cat = np.array( - list( - map( - tuple, - np.array( - [ - psf_file["X"], - psf_file["Y"], - psf_file["RA"], - psf_file["DEC"], - psf_file["E1_PSF_HSM"], - psf_file["E2_PSF_HSM"], - psf_file["SIGMA_PSF_HSM"], - psf_file["FLAG_PSF_HSM"], - psf_file["E1_STAR_HSM"], - psf_file["E2_STAR_HSM"], - psf_file["SIGMA_STAR_HSM"], - psf_file["FLAG_STAR_HSM"], - np.ones_like(psf_file["RA"], dtype=int) - * ccd_id, - ] - ).T.tolist(), - ) - ), - dtype=self._dt, - ) - cat_list.append(exp_cat) - - else: - l2g = Loc2Glob() - g2c = Glob2CCD(l2g) - new_ccd_id = np.array( - [ - int( - g2c.get_ccd_n( - psf_file["GLOB_POSITION_IMG_LIST"][ii, 0], - psf_file["GLOB_POSITION_IMG_LIST"][ii, 1], - ) - ) - for ii in range(len(psf_file)) - ] - ) - - # Local-to-CCD position: subtract each CCD's focal-plane shift. - new_x = np.zeros_like(psf_file[mod]) - new_y = np.zeros_like(psf_file[mod]) - - # The MCCD PSF_MOM_LIST/STAR_MOM_LIST columns come from the - # external mccd fit-validation code, which still measures HSM - # moments in the pixel frame; rotate them into world coordinates - # via the per-CCD WCS Jacobian. This rotation stays until mccd - # itself adopts use_sky_coords (see the module docstring). The - # in-repo PSFEx / MCCD-interpolation paths are already in world - # coordinates and are passed through unrotated. - new_e1_psf = np.zeros_like(psf_file[mod]) - new_e2_psf = np.zeros_like(psf_file[mod]) - new_sig_psf = np.zeros_like(psf_file[mod]) - new_e1_star = np.zeros_like(psf_file[mod]) - new_e2_star = np.zeros_like(psf_file[mod]) - new_sig_star = np.zeros_like(psf_file[mod]) - new_flag_psf = np.zeros_like(psf_file[mod]) - new_flag_star = np.zeros_like(psf_file[mod]) - for ccd_id in range(40): - m_ccd_id = new_ccd_id == ccd_id - if sum(m_ccd_id) == 0: - continue - - x_shift, y_shift = l2g.shift_coord(ccd_id) - - new_x[m_ccd_id] = ( - psf_file["GLOB_POSITION_IMG_LIST"][:, 0][m_ccd_id] - - x_shift - ) - new_y[m_ccd_id] = ( - psf_file["GLOB_POSITION_IMG_LIST"][:, 1][m_ccd_id] - - y_shift - ) - - header_file_path = ( - self._params["sub_dir_setools"] - + self._params["file_pattern_psfint"] - + f"{exp_name}-{ccd_id}.fits" - ) - try: - header_file = fits.getdata(header_file_path, 1) - except Exception: - continue - header = fits.Header.fromstring( - "\n".join(header_file[0][0]), sep="\n" - ) - wcs = galsim.AstropyWCS(header=header) - - g1_psf_tmp_l = [] - g2_psf_tmp_l = [] - sig_psf_tmp_l = [] - g1_star_tmp_l = [] - g2_star_tmp_l = [] - sig_star_tmp_l = [] - flag_psf_tmp_l = [] - flag_star_tmp_l = [] - - for obj in psf_file[m_ccd_id]: - try: - jac = wcs.jacobian( - world_pos=galsim.CelestialCoord( - ra=obj["RA_LIST"] * galsim.degrees, - dec=obj["DEC_LIST"] * galsim.degrees, - ) - ) - except Exception: - flag_star_tmp_l.append(16) - flag_psf_tmp_l.append(16) - g1_psf_tmp_l.append(0) - g2_psf_tmp_l.append(0) - sig_psf_tmp_l.append(0) - g1_star_tmp_l.append(0) - g2_star_tmp_l.append(0) - sig_star_tmp_l.append(0) - continue - g1_psf_tmp, g2_psf_tmp, sig_psf_tmp = transform_shape( - obj["PSF_MOM_LIST"], jac - ) - - g1_psf_tmp_l.append(g1_psf_tmp) - g2_psf_tmp_l.append(g2_psf_tmp) - sig_psf_tmp_l.append(sig_psf_tmp) - flag_psf_tmp_l.append(obj["PSF_MOM_LIST"][3]) - - g1_star_tmp, g2_star_tmp, sig_star_tmp = ( - transform_shape(obj["STAR_MOM_LIST"], jac) - ) - g1_star_tmp_l.append(g1_star_tmp) - g2_star_tmp_l.append(g2_star_tmp) - sig_star_tmp_l.append(sig_star_tmp) - flag_star_tmp_l.append(obj["STAR_MOM_LIST"][3]) - - new_e1_psf[m_ccd_id] = g1_psf_tmp_l - new_e2_psf[m_ccd_id] = g2_psf_tmp_l - new_sig_psf[m_ccd_id] = sig_psf_tmp_l - new_flag_psf[m_ccd_id] = flag_psf_tmp_l - new_e1_star[m_ccd_id] = g1_star_tmp_l - new_e2_star[m_ccd_id] = g2_star_tmp_l - new_sig_star[m_ccd_id] = sig_star_tmp_l - new_flag_star[m_ccd_id] = flag_star_tmp_l - - exp_cat = np.array( - list( - map( - tuple, - np.array( - [ - new_x, - new_y, - psf_file["RA_LIST"], - psf_file["DEC_LIST"], - new_e1_psf, - new_e2_psf, - new_sig_psf, - psf_file["PSF_MOM_LIST"][:, 3], - new_e1_star, - new_e2_star, - new_sig_star, - psf_file["STAR_MOM_LIST"][:, 3], - new_ccd_id, - psf_file["GLOB_POSITION_IMG_LIST"][:, 0], - psf_file["GLOB_POSITION_IMG_LIST"][:, 1], - ] - ).T.tolist(), - ) - ), - dtype=self._dt_mccd, - ) - cat_list.append(exp_cat) - - del psf_file - - if len(cat_list) == 0: - return - - # Finalize catalogue - patch_cat = np.concatenate(cat_list) - hdul = fits.HDUList() - hdul.append(fits.PrimaryHDU()) - hdul.append(fits.BinTableHDU(patch_cat)) - - # Write catalogue - hdul.writeto( - output_path, - overwrite=True, - ) - - del cat_list - del hdul - gc.collect() - - -def run_convert(*args): - - # Create instance - obj = Convert() - - obj.set_params_from_command_line(args) - obj.update_params() - - obj.run() - - -def main(argv=None): - """Main - - Main program - """ - if argv is None: - argv = sys.argv[1:] - run_convert(*argv) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py deleted file mode 100755 index 05e3d3bb9..000000000 --- a/scripts/python/create_star_cat.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script create_star_cat.py - -:Description: Create reference star catalogue for masking of -bright star halos and diffraction spikes - -:Authors: Axel Guinot, Martin Kilbinger - -""" - - -import os -import re -import sys - -from cs_util import args as cs_args -from cs_util import logging as cs_logging - -from astropy.io import fits - -from shapepipe.utilities.file_io import write_atomic -from shapepipe.utilities.focal_plane import ccd_center_and_radius, focal_plane_disc -from shapepipe.utilities.vizier import query_vizier as _query_vizier - - -# GSC 2.3 catalog ID -CDS_CAT_ID = "I/305/out" - - -def query_vizier(ra, dec, radius_arcmin): - return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) - - -def main(input_dir, output_dir, kind): - - file_list = os.listdir(input_dir) - - for f in file_list: - if "image" not in f: - continue - - img_number = re.split("image", os.path.splitext(f)[0])[1] - fpath = os.path.join(input_dir, f) - - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - if kind == "exp": - # One query covering the full MegaCam focal plane - ra, dec, radius_deg = focal_plane_disc(fpath) - radius = radius_deg * 60.0 - print( - f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" - ) - else: - # A single image: its own centre and half-diagonal. - ra, dec, radius_deg = ccd_center_and_radius(fits.getheader(fpath, 0)) - radius = radius_deg * 60.0 - - table = query_vizier(ra, dec, radius) - write_atomic(table, output_name) - - return 0 - - -def params_default(): - """Return default parameters, short options, types, and help strings.""" - _params = { - "input_dir": ".", - "output_dir": ".", - "kind": "exp", - } - _short_options = { - "input_dir": "-i", - "output_dir": "-o", - "kind": "-k", - } - _types = {} - _help_strings = { - "input_dir": "input directory containing image files; default is {}", - "output_dir": "output directory for star catalogues; default is {}", - "kind": "processing kind, 'exp' for full MegaCam focal plane, 'tile' for single image; default is {}", - } - return _params, _short_options, _types, _help_strings - - -if __name__ == "__main__": - - _params, _short_options, _types, _help_strings = params_default() - - options = cs_args.parse_options( - _params, - _short_options, - _types, - _help_strings, - ) - - cs_logging.log_command(sys.argv) - - main(options["input_dir"], options["output_dir"], options["kind"]) diff --git a/src/shapepipe/modules/mask_ext_package/__init__.py b/src/shapepipe/modules/mask_ext_package/__init__.py deleted file mode 100644 index 4e2123986..000000000 --- a/src/shapepipe/modules/mask_ext_package/__init__.py +++ /dev/null @@ -1,71 +0,0 @@ -"""MASK EXT MODULE. - -This package contains the module for ``mask_ext``. - -:Author: Cail Daley - -:Parent module: ``split_exp_runner`` (exposures) or ``get_images_runner`` / - ``uncompress_fits_runner`` (tiles), or None - -:Input: Single image (tile or single-exposure single-CCD) and, optionally, an - external instrument flag file - -:Output: Per-image pixel flag file - -Description -=========== - -This module produces the ShapePipe per-image pixel flag file -``_flag.fits`` by rasterizing an **external healsparse mask** onto -the image pixel grid, rather than generating masks internally (the job of the -``mask`` module). It is the ShapePipe consumer of the unified UNIONS healsparse -mask products (PhotoPipe footprint + bright stars, MaxiMask, manual galaxy -masks), merged and stored as ``healsparse.HealSparseMap`` files. - -For each image the module: - -1. Reads the image header into an ``astropy.wcs.WCS``. -2. Evaluates the pixel grid to (RA, Dec) in row chunks (bounded memory). -3. Queries ``HealSparseMap.get_values_pos`` at every pixel centre. -4. Maps healsparse bit values to ShapePipe flag values through the - config-driven ``BIT_FLAG_MAP``, producing an ``int16`` flag image. -5. Optionally sums in an external instrument flag image (``USE_EXT_FLAG``), - matching the combination semantics of the ``mask`` module. -6. Writes ``_flag.fits`` carrying the image WCS in its header. - -Everything downstream (SExtractor ``IMAFLAGS_ISO``, setools star selection, -vignetmaker, ngmix) consumes this artifact unchanged; the module is a drop-in -replacement for ``mask`` at the flag-image contract. - -The same module serves tiles and exposure CCDs: only the input image (hence -WCS) differs. The healsparse file, its bit meanings, and all flag values live -**only in config** — the mask products evolve and are swapped at zero code cost. - -Module-specific config file entries -=================================== - -MASK_PATH : str - Path to the healsparse mask file (``.hsp``/``.fits``); band-agnostic -BIT_FLAG_MAP : str - Mapping from healsparse pixel bit value to output flag value, formatted as - ``":, :, ..."`` (e.g. ``"64:1, 2048:2"``). A pixel - carrying several bits receives the bitwise-OR of the mapped flag values. -OFF_MAP_FLAG : int, optional - Flag value assigned to pixels that fall outside the healsparse map - footprint (sentinel pixels); default is ``0``. Off-footprint pixels are - usually unobserved, so a non-zero value flags them. -USE_EXT_FLAG : bool, optional - If ``True``, sum an external instrument flag file (given as the second - input) into the rasterized mask; default is ``False``. Needed for - exposures, where saturation / bleeding / bad columns arrive with the data. -HDU : int, optional - HDU of the external instrument flag FITS file; default is ``0`` -PREFIX : str, optional - Prefix prepended to the output file name base ``flag``; default is ``""`` -CHUNK_SIZE : int, optional - Number of image rows evaluated per chunk; default is chosen from the image - size to bound memory (``1e8`` px for tiles, ``1e7`` px for exposure CCDs) - -""" - -__all__ = ["mask_ext"] diff --git a/src/shapepipe/modules/mask_ext_package/mask_ext.py b/src/shapepipe/modules/mask_ext_package/mask_ext.py deleted file mode 100644 index c24aaabcd..000000000 --- a/src/shapepipe/modules/mask_ext_package/mask_ext.py +++ /dev/null @@ -1,333 +0,0 @@ -"""MASK EXT. - -This module contains a class to rasterize an external healsparse mask onto an -image pixel grid, producing the ShapePipe per-image pixel flag file. - -:Author: Cail Daley - -""" - -import re - -import numpy as np -from astropy import wcs - -from shapepipe.pipeline import file_io - -# Per-chunk pixel budget: chunk the image in row bands so that no more than -# this many pixel coordinates are held / queried at once. A tile is ~1e8 px and -# an exposure CCD ~1e7 px; this bound keeps peak memory to a single band. -DEFAULT_PIXEL_BUDGET = 10_000_000 - - -class MaskExt(object): - """Mask Ext. - - Rasterize an external healsparse mask onto an image pixel grid and write - the resulting ShapePipe flag file. - - Parameters - ---------- - image_path : str - Path to the image whose pixel grid and WCS define the output - mask_path : str - Path to the external healsparse mask file - bit_flag_map : dict - Mapping from healsparse pixel bit value (int) to output flag value - (int); a pixel carrying several bits gets the bitwise-OR of the mapped - flags - image_num : str - File number string, inserted into the output file name - output_dir : str - Path to the output directory - w_log : logging.Logger - Log file - off_map_flag : int, optional - Flag value for pixels outside the healsparse footprint (sentinel - pixels); default is ``0`` - path_external_flag : str, optional - Path to an external instrument flag file to sum into the mask; default - is ``None`` (not used) - image_prefix : str, optional - Prefix prepended to the output file name base ``flag``; specify - ``'none'`` or ``''`` for no prefix, default is ``''`` - outname_base : str, optional - Output file name base, default is ``flag`` - chunk_size : int, optional - Number of image rows evaluated per chunk; default is derived from the - image width and :data:`DEFAULT_PIXEL_BUDGET` - hdu : int, optional - HDU of the external instrument flag FITS file; default is ``0`` - - """ - - def __init__( - self, - image_path, - mask_path, - bit_flag_map, - image_num, - output_dir, - w_log, - off_map_flag=0, - path_external_flag=None, - image_prefix="", - outname_base="flag", - chunk_size=None, - hdu=0, - ): - self._image_path = image_path - self._mask_path = mask_path - self._bit_flag_map = bit_flag_map - self._img_number = image_num - self._output_dir = output_dir - self._w_log = w_log - self._off_map_flag = int(off_map_flag) - self._path_external_flag = path_external_flag - - if (image_prefix.lower() != "none") and (image_prefix != ""): - self._img_prefix = f"{image_prefix}_" - else: - self._img_prefix = "" - - self._outname_base = outname_base - self._chunk_size = chunk_size - self._hdu = hdu - - self._set_image_coordinates() - - @staticmethod - def parse_bit_flag_map(map_string): - """Parse Bit Flag Map. - - Parse the ``BIT_FLAG_MAP`` config string into a dictionary. - - Parameters - ---------- - map_string : str - Mapping formatted as ``":, :, ..."``, e.g. - ``"64:1, 2048:2"`` - - Returns - ------- - dict - Mapping from healsparse bit value (int) to output flag value (int) - - Raises - ------ - ValueError - If an entry is not of the form ``:`` - - """ - bit_flag_map = {} - for entry in map_string.split(","): - entry = entry.strip() - if not entry: - continue - if not re.fullmatch(r"\d+\s*:\s*\d+", entry): - raise ValueError( - f"Invalid BIT_FLAG_MAP entry '{entry}'; expected " - + "':'" - ) - bit, flag = (int(part) for part in entry.split(":")) - bit_flag_map[bit] = flag - if not bit_flag_map: - raise ValueError("BIT_FLAG_MAP is empty") - return bit_flag_map - - def _set_image_coordinates(self): - """Set Image Coordinates. - - Read the image header into a WCS and record the image shape, mirroring - ``Mask._set_image_coordinates``. - - """ - img = file_io.FITSCatalogue(self._image_path, hdu_no=0) - img.open() - self._header = img.get_header() - # get_data().shape is (n_y, n_x) - self._img_shape = img.get_data().shape - img.close() - del img - - self._wcs = wcs.WCS(self._header) - - def _default_chunk_size(self): - """Default Chunk Size. - - Rows per chunk such that one band holds at most - :data:`DEFAULT_PIXEL_BUDGET` pixels. - - Returns - ------- - int - Number of rows per chunk (at least 1) - - """ - n_x = self._img_shape[1] - return max(1, DEFAULT_PIXEL_BUDGET // n_x) - - def _map_bits_to_flags(self, bit_values, off_map): - """Map Bits to Flags. - - Translate healsparse bit values to output flag values through the - bit→flag mapping, OR-combining every matching bit, and assign the - off-map flag to sentinel pixels. - - Parameters - ---------- - bit_values : numpy.ndarray - Healsparse values queried at the pixel centres - off_map : numpy.ndarray - Boolean mask, ``True`` where the pixel lies outside the footprint - - Returns - ------- - numpy.ndarray - Output flag values (``int16``), same shape as ``bit_values`` - - """ - flags = np.zeros(bit_values.shape, dtype=np.int16) - for bit, flag in self._bit_flag_map.items(): - flags[(bit_values & bit) != 0] |= np.int16(flag) - if self._off_map_flag != 0: - flags[off_map] = np.int16(self._off_map_flag) - return flags - - def rasterize(self): - """Rasterize. - - Evaluate the pixel grid to (RA, Dec) in row chunks, query the - healsparse mask, and build the ``int16`` flag image. - - Returns - ------- - numpy.ndarray - The rasterized flag image, shape ``(n_y, n_x)``, dtype ``int16`` - - """ - # Lazy import: healsparse is an optional heavy dependency, imported at - # use rather than module load. - import healsparse - - hmap = healsparse.HealSparseMap.read(self._mask_path) - sentinel = hmap.sentinel - - # Mask products come in two flavours: integer bit-flag maps (per-band - # bits, e.g. 64 = r) and boolean maps (True = masked, e.g. the - # candide copy of the 2025 r-band mask). For a boolean map the only - # meaningful bit is 1 (True); any other bit silently selects nothing - # (``True & 64 == 0`` -> an all-clean flag image), so fail loudly. - if hmap.dtype == np.bool_: - bad_bits = [bit for bit in self._bit_flag_map if bit != 1] - if bad_bits: - raise ValueError( - f"Mask {self._mask_path} is a boolean healsparse map; " - + f"BIT_FLAG_MAP bits {bad_bits} would never match " - + "(use '1:' for boolean masks)" - ) - - n_y, n_x = self._img_shape - chunk_size = self._chunk_size or self._default_chunk_size() - - flag_image = np.zeros((n_y, n_x), dtype=np.int16) - # Column indices are shared across every row band. - x = np.arange(n_x) - - for y0 in range(0, n_y, chunk_size): - y1 = min(y0 + chunk_size, n_y) - yy, xx = np.meshgrid(np.arange(y0, y1), x, indexing="ij") - # WCS uses 0-based pixel coordinates here (origin=0). - ra, dec = self._wcs.all_pix2world(xx.ravel(), yy.ravel(), 0) - # Normalize RA into [0, 360) so the wrap at 0/360 is handled; - # healsparse expects lon in that range with lonlat=True. - ra = np.mod(ra, 360.0) - - bit_values = hmap.get_values_pos(ra, dec, lonlat=True) - off_map = bit_values == sentinel - - band = self._map_bits_to_flags(bit_values, off_map) - flag_image[y0:y1, :] = band.reshape(y1 - y0, n_x) - - return flag_image - - def _combine_external_flag(self, flag_image): - """Combine External Flag. - - Sum an external instrument flag image into the rasterized mask, - matching ``Mask._build_final_mask`` semantics (element-wise sum of the - two integer flag images). - - Parameters - ---------- - flag_image : numpy.ndarray - The rasterized flag image - - Returns - ------- - numpy.ndarray - The combined flag image (``int16``) - - """ - external_flag = file_io.FITSCatalogue( - self._path_external_flag, - hdu_no=self._hdu, - ) - external_flag.open() - ext_flag = external_flag.get_data()[:, :] - external_flag.close() - - return (flag_image + ext_flag).astype(np.int16, copy=False) - - def _output_path(self): - """Output Path. - - Full path of the output flag file, matching the ``mask`` module naming - (``_flag.fits``). - - Returns - ------- - str - Output file path - - """ - name = ( - f"{self._img_prefix}{self._outname_base}" - + f"{self._img_number}.fits" - ) - return f"{self._output_dir}/{name}" - - def make_mask(self): - """Make Mask. - - Rasterize the healsparse mask, optionally combine the external - instrument flag, and write the flag file carrying the image WCS. - - Returns - ------- - str - Path to the written flag file - - """ - flag_image = self.rasterize() - - if self._path_external_flag is not None: - flag_image = self._combine_external_flag(flag_image) - - output_path = self._output_path() - out = file_io.FITSCatalogue( - output_path, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - hdu_no=0, - ) - out.save_as_fits( - data=flag_image, - image=True, - image_header=self._wcs.to_header(), - ) - - self._w_log.info( - f"Wrote healsparse-derived flag file {output_path}" - ) - - return output_path diff --git a/src/shapepipe/modules/mask_ext_runner.py b/src/shapepipe/modules/mask_ext_runner.py deleted file mode 100644 index a43265d1b..000000000 --- a/src/shapepipe/modules/mask_ext_runner.py +++ /dev/null @@ -1,117 +0,0 @@ -"""MASK EXT RUNNER. - -Module runner for ``mask_ext``. - -:Author: Cail Daley - -""" - -from shapepipe.modules.mask_ext_package.mask_ext import MaskExt -from shapepipe.modules.module_decorator import module_runner - - -@module_runner( - version="1.0", - file_pattern=["image", "flag"], - file_ext=[".fits", ".fits"], - depends=["numpy", "astropy", "healsparse"], - numbering_scheme="_0", -) -def mask_ext_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Mask Ext Runner. - - Rasterize an external healsparse mask onto the input image pixel grid and - write the ShapePipe flag file. - - Notes - ----- - Only the image is strictly required: it supplies the pixel grid and WCS - onto which the healsparse mask is rasterized. Unlike the ``mask`` module, - no weight file is used — the healsparse mask already encodes the footprint, - and missing-data handling is not this module's concern. A second input, an - external instrument flag file, is consumed only when ``USE_EXT_FLAG`` is - ``True`` (needed for exposures, where saturation / bleeding / bad columns - arrive with the data). - - """ - n_inputs = len(input_file_list) - use_ext_flag = config.getboolean(module_config_sec, "USE_EXT_FLAG") if ( - config.has_option(module_config_sec, "USE_EXT_FLAG") - ) else False - - if use_ext_flag: - if n_inputs != 2: - raise ValueError( - f"Found {n_inputs} inputs but USE_EXT_FLAG is True, which " - + 'expects "image" and "flag" in the MASK_EXT_RUNNER section ' - + "of the config file." - ) - image_path = input_file_list[0] - ext_flag_name = input_file_list[1] - else: - if n_inputs != 1: - raise ValueError( - f"Found {n_inputs} inputs but USE_EXT_FLAG is False, which " - + 'expects only "image" in the MASK_EXT_RUNNER section of the ' - + "config file." - ) - image_path = input_file_list[0] - ext_flag_name = None - - # Path to the healsparse mask file - mask_path = config.getexpanded(module_config_sec, "MASK_PATH") - - # Bit -> flag mapping - bit_flag_map = MaskExt.parse_bit_flag_map( - config.get(module_config_sec, "BIT_FLAG_MAP") - ) - - # Flag value for pixels outside the healsparse footprint - if config.has_option(module_config_sec, "OFF_MAP_FLAG"): - off_map_flag = config.getint(module_config_sec, "OFF_MAP_FLAG") - else: - off_map_flag = 0 - - # HDU of the external instrument flag file - if config.has_option(module_config_sec, "HDU"): - hdu = config.getint(module_config_sec, "HDU") - else: - hdu = 0 - - # Output file name prefix - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - else: - prefix = "" - - # Rows per chunk (optional; default derived from image size) - if config.has_option(module_config_sec, "CHUNK_SIZE"): - chunk_size = config.getint(module_config_sec, "CHUNK_SIZE") - else: - chunk_size = None - - mask_inst = MaskExt( - image_path, - mask_path, - bit_flag_map, - file_number_string, - run_dirs["output"], - w_log, - off_map_flag=off_map_flag, - path_external_flag=ext_flag_name, - image_prefix=prefix.replace(" ", ""), - outname_base="flag", - chunk_size=chunk_size, - hdu=hdu, - ) - - mask_inst.make_mask() - - return None, None diff --git a/src/shapepipe/modules/mask_package/__init__.py b/src/shapepipe/modules/mask_package/__init__.py deleted file mode 100644 index bdfece65b..000000000 --- a/src/shapepipe/modules/mask_package/__init__.py +++ /dev/null @@ -1,185 +0,0 @@ -"""MASK MODULE. - -This package contains the module for ``mask``. - -:Author: Axel Guinot - -:Parent module: ``split_exp_runner`` or None - -:Input: Single-exposure single-CCD image, weight file, flag file (optional), - and star catalogue (optional) - -:Output: Single-exposure single-CCD flag files - -Description -=========== - -This module creates masks for bright stars, diffraction spikes, deep sky -objects (from the Messier and NGC catalogues), borders, and other artifacts. If -a flag file is given as input, for example from pre-processing, the mask that -is created by this module is joined with the mask from this external flag file. -In this case the config flag ``USE_EXT_FLAG`` needs to be set to ``True``. To -distinguish the newly created output flag file from the input ones, a prefix -can added as specificed by the config entry ``PREFIX``. - -An NGC catalogue with positions, sizes, and types is provided with -``shapepipe``, -`source `_. - -Masked pixels of different mask types are indicated by integers, which -conveniently are powers of two such that they can be combined bit-wise. - -To mask bright stars, this module either creates a star catalogue from the -online -`guide star catalogue `_ -database relevant to the the footprint. This is done by calling a CDs -(Centre de Données astronomique de Strasbourg) -`client program `_. -Note that this requires online access, -which in some cases is not granted on compute nodes of a cluster. In this case, -set the config flag ``USE_EXT_STAR = False``. Alternatively, a star -catalogue can be created before running this module via the script -``create_star_cat``. During the processing of this module, this star catalogue -is read from disk, with ``USE_SET_STAR = True``. - -The masking is done with the software ``WeightWatcher`` :cite:`marmo:08`, -which is installed by ``ShapePipe`` by default. - -Module-specific config file entries -=================================== - -USE_EXT_FLAG : bool - Use external flag file to join with the mask created here; - if ``True`` flag file needs to be given on input -USE_EXT_STAR : bool - Read external star catalogue instead of creating one during the - call of this module; - if ``True`` star catalogue file needs to be given on input -MASK_CONFIG_PATH : str - Path to mask config file -HDU : int, optional - HDU of external flag FITS file; the default value is ``0`` -PREFIX : str, optional - Prefix to be appended to output file name ``flag``; - helps to distinguish the file patterns of newly created and external - mask files -CHECK_EXISTING_DIR : str, optional - If given, search this directory for existing mask files; the - corresponding images will then not be processed - -Mask config file -================ - -An additional configuration file is used by the mask module, its path is -``MASK_CONFIG_PATH`` in the module config section, see above. The following -describes the config file sections and their entries. - -[PROGRAM_PATH] --------------- - -WW_PATH : str, optional - Full path to the WeightWatcher executable (``ww``) on the system ; if not - set the version controlled WeightWatcher installation in the ShapePipe - environment will be used -WW_CONFIG_FILE : str - Path to the WeightWatcher configuration file -CDSCLIENT_PATH : str, optional - Path to CDS client executable; required if ``USE_EXT_STAR = False`` - -[BORDER_PARAMETERS] -------------------- - -BORDER_MAKE : bool - Create mask around borders if ``True`` -BORDER_WIDTH : int - Width of border mask in pixels -BORDER_FLAG_VALUE : int - Border mask pixel value, power of 2 - -[HALO_PARAMETERS] ------------------ - -HALO_MAKE : bool - Create mask for halos of bright stars if ``True`` -HALO_MASKMODEL_PATH : str - Path to halo mask geometry (``.reg`` file) -HALO_MAG_LIM : float - Faint stellar magnitude limit for halo mask -HALO_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and halo mask size -HALO_MAG_PIVOT : float - Pivot stellar magnitude -HALO_FLAG_VALUE : int - Halo mask pixel value, power of 2 -HALO_REG_FILE : str - Output halo mask ``.reg`` file - -[SPIKE_PARAMETERS] ------------------- - -SPIKE_MAKE : bool - Create mask for diffraction spikes of bright stars if ``True`` -SPIKE_MASKMODEL_PATH : str - Path to diffraction spike geometry (``.reg`` file) -SPIKE_MAG_LIM : - Faint stellar magnitude limit for spike mask -SPIKE_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and spike mask size -SPIKE_MAG_PIVOT : float - Pivot stellar magnitude -SPIKE_FLAG_VALUE : int - Diffraction spike pixel value, power of two -SPIKE_REG_FILE : str - Output spike mask ``.reg`` file - -[MESSIER_PARAMETERS] --------------------- - -MESSIER_MAKE : bool - Create mask around Messier objects if ``True`` -MESSIER_CAT_PATH : str - Path to Messier catalogue -MESSIER_SIZE_PLUS : float - Fraction to increase Messier mask -MESSIER_FLAG_VALUE : int - Messier mask pixel value, power of 2 - -[NGC_PARAMETERS] --------------------- - -NGC_MAKE : bool - Create mask around NGC objects if ``True`` -NGC_CAT_PATH : str - Path to NGC catalogue -NGC_SIZE_PLUS : float - Fraction to increase NGC mask -NGC_FLAG_VALUE : int - NGC mask pixel value, power of 2 - -[MD_PARAMETERS] ---------------- - -MD_MAKE : bool - Account for missing data (zero-valued pixels) if ``True`` -MD_THRESH_FLAG : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked as flagged -MD_THRESH_REMOVE : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked for removal -MD_REMOVE : bool - Image is removed if marked for removal - -[OTHER] -------- - -TEMP_DIRECTORY : str - Path to temporary dictionary -KEEP_INDIVIDUAL_MASK : bool - Keep individual masks in addition to merged mask file -KEEP_REG_FILE : bool - Keep ``.reg`` mask file - -""" - -__all__ = ["mask"] diff --git a/src/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py deleted file mode 100644 index ba6d586ad..000000000 --- a/src/shapepipe/modules/mask_package/mask.py +++ /dev/null @@ -1,1271 +0,0 @@ -"""MASK. - -This module contains a class to create star mask for an image. - -:Authors: Axel Guinot, Martin Kilbinger - -""" - -import os -import re - -import numpy as np -from astropy import units, wcs -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.table import Table - -from shapepipe.pipeline import file_io -from shapepipe.pipeline.config import CustomParser -from shapepipe.pipeline.execute import execute -from shapepipe.utilities.file_system import mkdir -from shapepipe.utilities.vizier import query_vizier - - -class Mask(object): - """Mask. - - Class to create mask based on a star catalogue. - - Parameters - ---------- - image_path : str - Path to image (FITS format) - weight_path : str - Path to the weight image (FITS format) - image_prefix : str - Prefix to input image name, specify as ``'none'`` for no prefix - image_num : str - File number identified - config_filepath : str - Path to the ``.mask`` config file - output_dir : str - Path to the output directory - w_log : logging.Logger - Log file - path_external_flag : str, optional - Path to external flag file, default is ``None`` (not used) - outname_base : str, optional - Output file name base, default is ``flag`` - check_existing_dir : str, optional - If not ``None`` (default), search path for existing mask files - star_cat_path : str, optional - Path to external star catalogue, default is ``None`` (not used; - instead the star catalogue is produced on the fly at run time) - hdu : int, optional - HDU number, default is ``0`` - - """ - - def __init__( - self, - image_path, - weight_path, - image_prefix, - image_num, - config_filepath, - output_dir, - w_log, - path_external_flag=None, - outname_base="flag", - check_existing_dir=None, - star_cat_path=None, - hdu=0, - ): - - # Path to the image to mask - self._image_fullpath = image_path - - # Path to the weight associated to the image - self._weight_fullpath = weight_path - - # Input image prefix - if (image_prefix.lower() != "none") and (image_prefix != ""): - self._img_prefix = f"{image_prefix}_" - else: - self._img_prefix = "" - - # File number identified - self._img_number = image_num - - # Path to mask config file - self._config_filepath = config_filepath - - # Path to the output directory - self._output_dir = output_dir - - # Log file - self._w_log = w_log - - # Path to an external flag file - self._path_external_flag = path_external_flag - - # Output file base name - self._outname_base = outname_base - - # Search path for existing mask files - self._check_existing_dir = check_existing_dir - - # Set external star catalogue path if given - if star_cat_path is not None: - self._star_cat_path = star_cat_path - - self._hdu = hdu - - # Read mask config file - self._get_config() - - # Set parameters needed for the star detection - self._set_image_coordinates() - - # Set error flag - self._err = False - - # Guide Star Catalogue parameters - #self._CDS_cat_ID = "I/271/out" # GSC 2.2, does not have Fmag - self._CDS_cat_ID = "I/305/out" # GSC 2.3 - - # Keys in CDS astroquery result - self._cds_keys = ["GSC2.3", "RAJ2000", "DEJ2000", "Fmag", "jmag", "Vmag", "Nmag", "Class"] - - # Minimal scaling for halo and spike polygon templates - self._scaling_min = 0.1 - - def _get_config(self): - """Get Config. - - Read the config file and set parameters. - - Raises - ------ - ValueError - If config file name is ``None`` - IOError - If config file not found - - """ - if self._config_filepath is None: - raise ValueError("No path to config file given") - - if not os.path.exists(self._config_filepath): - raise IOError(f'Config file "{self._config_filepath}" not found') - - conf = CustomParser() - conf.read(self._config_filepath) - - self._config = { - "PATH": {}, - "BORDER": {}, - "HALO": {}, - "SPIKE": {}, - "MESSIER": {}, - "NGC": {}, - "MD": {}, - } - - if conf.has_option("PROGRAM_PATH", "WW_PATH"): - self._config["PATH"]["WW"] = conf.getexpanded( - "PROGRAM_PATH", "WW_PATH" - ) - else: - self._config["PATH"]["WW"] = "weightwatcher" - self._config["PATH"]["WW_configfile"] = conf.getexpanded( - "PROGRAM_PATH", "WW_CONFIG_FILE" - ) - if conf.has_option("PROGRAM_PATH", "CDSCLIENT_PATH"): - self._config["PATH"]["CDSclient"] = conf.getexpanded( - "PROGRAM_PATH", "CDSCLIENT_PATH" - ) - elif self._star_cat_path is not None: - self._config["PATH"]["star_cat"] = self._star_cat_path - else: - raise ValueError( - "Either [PROGRAM_PATH]:CDSCLIENT_PATH in the mask config file " - + " or a star catalogue as module input needs to be present" - ) - - self._config["PATH"]["temp_dir"] = self._get_temp_dir_path( - conf.getexpanded("OTHER", "TEMP_DIRECTORY") - ) - self._config["BORDER"]["make"] = conf.getboolean( - "BORDER_PARAMETERS", "BORDER_MAKE" - ) - if self._config["BORDER"]["make"]: - self._config["BORDER"]["width"] = conf.getint( - "BORDER_PARAMETERS", "BORDER_WIDTH" - ) - self._config["BORDER"]["flag"] = conf.get( - "BORDER_PARAMETERS", "BORDER_FLAG_VALUE" - ) - - for mask_shape in ["HALO", "SPIKE"]: - - self._config[mask_shape]["make"] = conf.getboolean( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAKE", - ) - self._config[mask_shape]["individual"] = conf.getboolean( - "OTHER", "KEEP_INDIVIDUAL_MASK" - ) - - if self._config[mask_shape]["make"]: - - self._config[mask_shape]["maskmodel_path"] = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MASKMODEL_PATH", - ) - self._config[mask_shape]["mag_lim"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_LIM", - ) - self._config[mask_shape]["scale_factor"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_SCALE_FACTOR", - ) - self._config[mask_shape]["mag_pivot"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_PIVOT", - ) - self._config[mask_shape]["flag"] = conf.getint( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_FLAG_VALUE", - ) - - if conf.getboolean("OTHER", "KEEP_REG_FILE"): - reg_file = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_REG_FILE", - ) - self._config[mask_shape]["reg_file"] = ( - f'{self._config["PATH"]["temp_dir"]}/' - + f'{re.split(".reg", reg_file)[0]}' - + f"{self._img_number}.reg" - ) - else: - self._config[mask_shape]["reg_file"] = None - - for mask_type in ["MESSIER", "NGC"]: - - self._config[mask_type]["make"] = conf.getboolean( - f"{mask_type}_PARAMETERS", f"{mask_type}_MAKE" - ) - - if self._config[mask_type]["make"]: - self._config[mask_type]["cat_path"] = conf.getexpanded( - f"{mask_type}_PARAMETERS", - f"{mask_type}_CAT_PATH", - ) - self._config[mask_type]["size_plus"] = conf.getfloat( - f"{mask_type}_PARAMETERS", - f"{mask_type}_SIZE_PLUS", - ) - self._config[mask_type]["flag"] = conf.getint( - f"{mask_type}_PARAMETERS", - f"{mask_type}_FLAG_VALUE", - ) - - self._config["MD"]["make"] = conf.getboolean("MD_PARAMETERS", "MD_MAKE") - - if self._config["MD"]["make"]: - self._config["MD"]["thresh_flag"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_FLAG" - ) - self._config["MD"]["thresh_remove"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean( - "MD_PARAMETERS", "MD_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean("MD_PARAMETERS", "MD_REMOVE") - - def _set_image_coordinates(self): - """Set Image Coordinates. - - Compute the image coordinates for matching with the star catalogue - and star mask. - - """ - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - self._header = img.get_header() - img_shape = img.get_data().shape - img.close() - del img - - self._wcs = wcs.WCS(self._header) - - # Compute field center - - # Note: get_data().shape corresponds to (n_y, n_x) - pix_center = [img_shape[1] / 2.0, img_shape[0] / 2.0] - wcs_center = self._wcs.all_pix2world([pix_center], 1)[0] - self._fieldcenter = {} - self._fieldcenter["pix"] = np.array(pix_center) - self._fieldcenter["wcs"] = SkyCoord( - ra=wcs_center[0], dec=wcs_center[1], unit="deg" - ) - - # Get the four corners of the image - corners = self._wcs.calc_footprint() - self._corners_sc = SkyCoord( - ra=corners[:, 0] * units.degree, - dec=corners[:, 1] * units.degree, - ) - - # Compute image radius = image diagonal - self._img_radius = self._get_image_radius() - - def make_mask(self): - """Make Mask. - - Main function to create the mask. - - """ - output_file_name = ( - f"{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - if os.path.exists(f"{self._check_existing_dir}//{output_file_name}"): - return None, None - - if self._config["MD"]["make"]: - self.missing_data() - - if self._config["HALO"]["make"] or self._config["SPIKE"]["make"]: - stars = self.find_stars( - np.array( - [ - self._fieldcenter["wcs"].ra.value, - self._fieldcenter["wcs"].dec.value, - ] - ), - radius=self._img_radius, - ) - - if not self._err: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._create_mask( - stars=stars, - types=_type, - mag_limit=self._config[_type]["mag_lim"], - scale_factor=self._config[_type]["scale_factor"], - mag_pivot=self._config[_type]["mag_pivot"], - ) - - if not self._err: - mask_name = [] - if self._config["HALO"]["make"] and self._config["SPIKE"]["make"]: - self._exec_WW(types="ALL") - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}halo_spike_flag' - + f"{self._img_number}.fits" - ) - mask_name.append(None) - else: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._exec_WW(types=_type) - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}' - + f"{_type.lower()}_flag{self._img_number}.fits" - ) - else: - mask_name.append(None) - - masks_internal = {} - if not self._err: - if self._config["BORDER"]["make"]: - masks_internal["BORDER"] = self.mask_border( - width=self._config["BORDER"]["width"] - ) - - if not self._err: - for _type in ("MESSIER", "NGC"): - if self._config[_type]["make"]: - masks_internal[_type] = self.mask_dso( - self._config[_type]["cat_path"], - size_plus=self._config[_type]["size_plus"], - flag_value=self._config[_type]["flag"], - obj_type=_type, - ) - - if not self._err: - try: - im_pass = self._config["MD"]["im_remove"] - except Exception: - im_pass = True - - if not self._err: - path_external_flag = self._path_external_flag - - if not self._err: - if im_pass: - final_mask = self._build_final_mask( - path_mask1=mask_name[0], - path_mask2=mask_name[1], - masks_internal=masks_internal, - path_external_flag=path_external_flag, - ) - - if not self._config["HALO"]["individual"]: - if mask_name[0] is not None: - self._rm_fits1_stdout, self._rm_fits1_stderr = execute( - f"rm {mask_name[0]}" - ) - if mask_name[1] is not None: - self._rm_fits2_stdout, self._rm_fits2_stderr = execute( - f"rm {mask_name[1]}" - ) - - output_file_name = ( - f"{self._output_dir}/{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - - self._mask_to_file( - input_mask=final_mask, - output_fullpath=output_file_name, - ) - - # Handle stdout / stderr - # _CDS_stdout/_CDS_stderr are only set when find_stars ran, i.e. - # when HALO_MAKE or SPIKE_MAKE is True (False for image sims) - general_stdout = "" - general_stderr = "" - if hasattr(self, "_CDS_stdout"): - general_stdout += f"\nCDSClient\n{self._CDS_stdout}" - if self._CDS_stderr != "": - general_stderr += f"\nCDSClient\n{self._CDS_stderr}" - if hasattr(self, "_WW_stdout") or hasattr(self, "_WW_stdout"): - general_stdout += f"\n\nWeightWatcher\n{self._WW_stdout}" - if self._WW_stderr != "": - general_stderr += f"\n\nWeightWatcher\n{self._WW_stderr}" - if hasattr(self, "_rm_reg_stderr") or hasattr(self, "_rm_reg_stdout"): - general_stdout += f"\n\nrm reg file\n{self._rm_reg_stdout}" - if self._rm_reg_stderr != "": - general_stderr += f"\n\nrm reg file\n{self._rm_reg_stderr}" - if hasattr(self, "_rm_fits1_stderr") or hasattr( - self, "_rm_fits1_stdout" - ): - general_stdout += f"\n\nrm fits1 file\n{self._rm_fits1_stdout}" - if self._rm_fits1_stderr != "": - general_stderr += f"\n\nrm fits1 file\n{self._rm_fits1_stderr}" - if hasattr(self, "_rm_fits2_stderr") or hasattr( - self, "_rm_fits2_stdout" - ): - general_stdout += f"\n\nrm fits2 file\n{self._rm_fits2_stdout}" - if self._rm_fits2_stderr != "": - general_stderr += f"\n\nrm fits2 file\n{self._rm_fits2_stderr}" - - return general_stdout, general_stderr - - def find_stars(self, position, radius): - """Find Stars. - - Return GSC (Guide Star Catalog) objects for a field with center - (RA, Dec) and radius :math:`r`. - - Parameters - ---------- - position : numpy.ndarray - Position of the center of the field - radius : float - Radius in which the query is done (in arcmin) - - Returns - ------- - dict - Star dictionnary for GSC objects in the field - - Raises - ------ - ValueError - For invalid configuration options - - """ - if "star_cat" in self._config["PATH"]: - self._CDS_stdout = Table.read(self._config["PATH"]["star_cat"]) - else: - # For some exposures, Vizier returned empty star list if input position - # is not single (? or double) precision - p = np.array(position, dtype='double') - - coord = SkyCoord(ra=p[0] * units.deg, dec=p[1] * units.deg, frame="icrs") - - self._CDS_stdout = query_vizier(p[0], p[1], radius, self._CDS_cat_ID) - - self._CDS_stderr = "" - - return self._make_star_cat(self._CDS_stdout) - - def mask_border(self, width=100, flag_value=4): - """Create Mask Border. - - Mask ``width`` pixels around the image. - - Parameters - ---------- - width : int - Width of the mask mask border - flag_value : int - Value of the flag for the border (power of 2) - - Returns - ------- - numpy.ndarray - Array containing the mask - - Raises - ------ - ValueError - If ``width`` is ``None`` - - """ - if width is None: - raise ValueError("Width for border mask not provided") - - # Note that python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - flag[0:width, :] = flag_value - flag[-width:, :] = flag_value - flag[:, 0:width] = flag_value - flag[:, -width:] = flag_value - - return flag - - def mask_dso( - self, - cat_path, - size_plus=0.1, - flag_value=8, - obj_type="Messier", - ): - """Mask DSO. - - Create a circular patch for deep-sky objects (DSOs), e.g. - Messier or NGC objects. - - Parameters - ---------- - cat_path : str - Path to the deep-sky catalogue - size_plus : float - Increase the size of the mask by this factor - (e.g. ``0.1`` means 10%) - flag_value : int - Value of the flag, some power of 2 - obj_type : {'Messier', 'NGO'}, optional - Object type - - Returns - ------- - numpy.ndarray or ``None`` - If no deep-sky objects are found in the field return ``None`` and - the flag map - - Raises - ------ - ValueError - If ``size_plus`` is negative - ValueError - If ``cat_path`` is ``None`` - - """ - if size_plus < 0: - raise ValueError( - "deep-sky mask size increase variable cannot be negative" - ) - - if cat_path is None: - raise ValueError("Path to deep-sky object catalogue not provided") - - m_cat, header = fits.getdata(cat_path, header=True) - - unit_ra = file_io.get_unit_from_fits_header(header, "ra") - unit_dec = file_io.get_unit_from_fits_header(header, "dec") - m_sc = SkyCoord( - ra=m_cat["ra"] * unit_ra, - dec=m_cat["dec"] * unit_dec, - ) - - unit_size_X = file_io.get_unit_from_fits_header(header, "size_X") - unit_size_Y = file_io.get_unit_from_fits_header(header, "size_Y") - - # Loop through all deep-sky objects and check whether the object's - # disc overlaps the image footprint - indices = [] - size_max_deg = [] - for idx, m_obj in enumerate(m_cat): - - # DSO size - # r = max(m_obj['size']) * units.arcmin - r = max( - m_obj["size_X"] * unit_size_X, - m_obj["size_Y"] * unit_size_Y, - ) - r_deg = r.to(units.degree) - size_max_deg.append(r_deg) - - # Add index to list if the DSO disc overlaps the image: - # distance between DSO centre and image centre smaller than - # DSO radius plus image half-diagonal. (Testing only the image - # corners against the DSO radius, as done previously, misses - # objects that are smaller than the image and lie away from - # the corners.) - dist = self._fieldcenter["wcs"].separation(m_sc[idx]) - if dist < r_deg + self._img_radius * units.arcmin: - indices.append(idx) - - self._w_log.info( - f"Found {len(indices)} {obj_type} objects overlapping with" " image" - ) - - if len(indices) == 0: - # No closeby deep-sky object found - return None - - # Compute number of DSO center coordinates in footprint, for logging - # purpose only - n_dso_center_in_footprint = 0 - for idx in indices: - in_img = self._wcs.footprint_contains(m_sc[idx]) - self._w_log.info( - "(obj_type, ra, dec, in_img) = " - + f"({obj_type}, " - + f'{m_cat["ra"][idx]}, ' - + f'{m_cat["dec"][idx]}, ' - + f"{in_img})" - ) - - # Note: python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - nx = self._fieldcenter["pix"][0] * 2 - ny = self._fieldcenter["pix"][1] * 2 - for idx in indices: - m_center = np.hstack( - self._wcs.all_world2pix( - m_cat["ra"][idx], - m_cat["dec"][idx], - 0, - ) - ) - r_pix = ( - size_max_deg[idx].to(units.deg).value - * (1 + size_plus) - / np.abs(self._wcs.pixel_scale_matrix[0][0]) - ) - - # The following accounts for deep-sky centers outside of image, - # without creating masks for coordinates out of range - y_c, x_c = np.ogrid[0:ny, 0:nx] - mask_tmp = (x_c - m_center[0]) ** 2 + ( - y_c - m_center[1] - ) ** 2 <= r_pix**2 - - flag[mask_tmp] = flag_value - - return flag - - def missing_data(self): - """Find Missing Data. - - Look for zero-valued pixels in image. Flag if their relative number - is larger than a threshold. - """ - # Open image - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - - # Get total number of pixels - im_shape = img.get_data().shape - tot = float(im_shape[0] * im_shape[1]) - - # Compute number and ratio of missing data (zero-valued pixels) - missing = float(len(np.where(img.get_data() == 0.0)[0])) - self._ratio = missing / tot - - # Mark image as to be flagged if ratio larger than 'flag' threshold - if self._ratio >= self._config["MD"]["thresh_flag"]: - self._config["MD"]["im_flagged"] = True - else: - self._config["MD"]["im_flagged"] = False - - # Mark image as to be removed if flag is True and - # ratio large than 'remove' threshold. - # Reset all other mask 'make' flags to False (no other mask needs - # to be created) - if self._config["MD"]["remove"]: - if self._ratio >= self._config["MD"]["thresh_remove"]: - self._config["MD"]["im_remove"] = True - for idx in ["HALO", "SPIKE", "MESSIER", "BORDER"]: - self._config[idx]["make"] = False - else: - self._config["MD"]["im_remove"] = False - - img.close() - - def sphere_dist(self, position1, position2): - """Compute Spherical Distance. - - Compute spherical distance between 2 points. - - Parameters - ---------- - position1 : numpy.ndarray - [x,y] first point (in pixels) - position2 : numpy.ndarray - [x,y] second point (in pixels) - - Returns - ------- - float - The distance in degrees. - - Raises - ------ - ValueError - If input positions are not Numpy arrays - - """ - if ( - type(position1) is not np.ndarray - or type(position2) is not np.ndarray - ): - raise ValueError("Object coordinates need to be a numpy.ndarray") - - p1 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position1[0], position1[1], 1) - ) - p2 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position2[0], position2[1], 1) - ) - - dTheta = p1 - p2 - dLong = dTheta[0] - dLat = dTheta[1] - - dist = 2 * np.arcsin( - np.sqrt( - np.sin(dLat / 2.0) ** 2.0 - + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(dLong / 2.0) ** 2.0 - ) - ) - - return dist * (180.0 / np.pi) * 3600.0 - - def _get_image_radius(self, center=None): - """Get Image Radius. - - Compute the diagonal distance of the image in arcmin. - - Parameters - ---------- - center : numpy.ndarray, optional - Coordinates of the center of the image (in pixels) - - Returns - ------- - float - The diagonal distance of the image in arcmin - - Raises - ------ - TypeError - If centre is not a Numpy array - - """ - if center is None: - return ( - self.sphere_dist(self._fieldcenter["pix"], np.zeros(2)) / 60.0 - ) - - else: - if isinstance(center, np.ndarray): - return self.sphere_dist(center, np.zeros(2)) / 60.0 - else: - raise TypeError( - "Image center coordinates has to be a numpy.ndarray" - ) - - def _make_star_cat(self, CDSclient_output): - """Make Star Catalogue. - - Create a dictionary from an astroquery request. - - Parameters - ---------- - CDSclient_output : str - Output astroquery - - Returns - ------- - dict - Star dictionary containing all information - - """ - header = [] - stars = {} - - for key in self._cds_keys: - stars[key] = CDSclient_output[key] - - return stars - - def _create_mask( - self, - stars, - types="HALO", - mag_limit=18.0, - mag_pivot=13.8, - scale_factor=0.3, - ): - """Create Mask. - - Apply mask from model to stars and save into DS9 region file. - - Parameters - ---------- - stars : dict - Stars dictionary (output of ``find_stars``) - types : {'HALO', 'SPIKE'}, optional - Type of mask, options are ``HALO`` or ``SPIKE`` - mag_limit : float, optional - Faint magnitude limit for mask, default is ``18.0`` - mag_pivot : float, optional - Pivot magnitude for the model, default is ``13.8`` - scale_factor : float, optional - Scaling for the model, default is ``0.3`` - - Raises - ------ - ValueError - If no star catalogue is provided - ValueError - If an invalid option is provided for type - - """ - if stars is None: - raise ValueError("Star catalogue dictionary not provided") - - if types not in ("HALO", "SPIKE"): - raise ValueError('Mask types need to be in ["HALO", "SPIKE"]') - - if self._config[types]["reg_file"] is None: - reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - else: - reg = self._config[types]["reg_file"] - - mask_model = np.loadtxt( - self._config[types]["maskmodel_path"] - ).transpose() - mask_reg = open(reg, "w") - - stars_used = [[], [], []] - - """ - star_zip = zip( - stars["RA(J2000)"], - stars["Dec(J2000)"], - stars["Fmag"], - stars["Jmag"], - stars["Vmag"], - stars["Nmag"], - stars["Clas"], - ) - """ - - # Get keys without object name - keys_to_use = self._cds_keys[1:] - star_zip = zip(*(stars[k] for k in keys_to_use)) - - for ra, dec, Fmag, Jmag, Vmag, Nmag, clas in star_zip: - # Compute mean magnitude over the available (finite) bands. - # Missing GSC bands are NaN and must be excluded: a single NaN - # would make the mean NaN and silently fail the - # ``mag < mag_limit`` test below, leaving bright stars with - # incomplete photometry (the ones that most need masking) - # unmasked. - mags = [ - band - for band in (Fmag, Jmag, Vmag, Nmag) - if band is not None and np.isfinite(band) - ] - if len(mags) > 0: - mag = sum(mags) / len(mags) - else: - mag = None - self._w_log.info( - f"No finite {types} magnitude for star at ra={ra} " - + f"dec={dec}; object not masked" - ) - - if ( - ra is not None - and dec is not None - and mag is not None - and clas is not None - ): - if (mag < mag_limit) and (clas == 0): - scaling = 1.0 - scale_factor * (mag - mag_pivot) - if scaling < self._scaling_min: - scaling = self._scaling_min - pos = self._wcs.all_world2pix(ra, dec, 0) - stars_used[0].append(pos[0]) - stars_used[1].append(pos[1]) - stars_used[2].append(scaling) - - for idx in range(len(stars_used[0])): - poly = "polygon(" - for x, y in zip(mask_model[0], mask_model[1]): - angle = np.arctan2(y, x) - ll = stars_used[2][idx] * np.sqrt(x**2 + y**2) - xnew = ll * np.cos(angle) - ynew = ll * np.sin(angle) - poly = ( - f"{poly}{str(stars_used[0][idx] + xnew + 0.5)} " - + f"{str(stars_used[1][idx] + ynew + 0.5)} " - ) - poly = f"{poly})\n" - mask_reg.write(poly) - - mask_reg.close() - - def _exec_WW(self, types="HALO"): - """Execute WeightWatcher. - - Execute WeightWatcher to transform ``.reg`` to ``.fits`` flag map. - - Parameters - ---------- - types : {'HALO', 'SPIKE', 'ALL'}, optional - Type of WeightWatcher execution, options are ``HALO``, - ``SPIKE`` or ``ALL`` - - Raises - ------ - BaseCatalogue.CatalogFileNotFound - If catalogue file not found - - """ - if types in ("HALO", "SPIKE"): - - default_reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - default_out = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}_flag' - + f"{self._img_number}.fits" - ) - - if self._config[types]["reg_file"] is None: - reg = default_reg - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute(f"rm {reg}") - - - else: - reg = self._config[types]["reg_file"] - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - - elif types == "ALL": - - default_reg = [ - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo{self._img_number}.reg" - ), - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"spike{self._img_number}.reg" - ), - ] - default_out = ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo_spike_flag{self._img_number}.fits" - ) - - if self._config["HALO"]["reg_file"] is None: - reg = default_reg - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute( - f"rm {reg[0]} {reg[1]}" - ) - else: - reg = [ - self._config["HALO"]["reg_file"], - self._config["SPIKE"]["reg_file"], - ] - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - else: - raise ValueError("Types must be in ['HALO','SPIKE','ALL']") - - if (self._WW_stderr != "") or (self._rm_reg_stderr != ""): - self._err = True - - def _build_final_mask( - self, - path_mask1, - path_mask2=None, - masks_internal=None, - path_external_flag=None, - ): - """Create Final Mask. - - Create the final mask by combining the individual masks. - - Parameters - ---------- - path_mask1 : str - Path to a mask (FITS format) - path_mask2 : str, optional - Path to a mask (FITS format) - masks_internal : dict, optional - Internally created masks - path_external_flag : str, optional - Path to an external flag file - - Returns - ------- - numpy.ndarray - Array containing the final mask - - Raises - ------ - ValueError - If all masks are of type ``None`` - TypeError - If border is not a Numpy array - TypeError - If Messier mask is not a Numpy array - - """ - final_mask = None - - if path_mask1 is None and path_mask2 is None and not masks_internal: - raise ValueError( - "No paths to mask files containing halos and/or spikes," - + " borders, or deep-sky objects provided" - ) - - if path_mask1 is not None: - mask1 = file_io.FITSCatalogue(path_mask1, hdu_no=self._hdu) - mask1.open() - dat = mask1.get_data() - final_mask = dat[:, :] - - if path_mask2 is not None: - mask2 = file_io.FITSCatalogue(path_mask2, hdu_no=self._hdu) - mask2.open() - if final_mask is not None: - final_mask += mask2.get_data()[:, :] - else: - final_mask = mask2.get_data()[:, :] - - for typ in masks_internal: - if masks_internal[typ] is not None: - if type(masks_internal[typ]) is np.ndarray: - if final_mask is not None: - final_mask += masks_internal[typ] - else: - final_mask = masks_internal[typ] - else: - raise TypeError( - f"internally created mask of type {typ} " - + "has to be numpy.ndarray" - ) - - if path_external_flag is not None: - external_flag = file_io.FITSCatalogue( - path_external_flag, - hdu_no=self._hdu, - ) - external_flag.open() - if final_mask is not None: - final_mask = final_mask.astype(np.int16, copy=False) - try: - ext_flag = external_flag.get_data() - except: - self._w_log.info( - "Problem while getting external flag data. Check" - + f" whether file {path_external_flag} is not corrupt" - ) - raise - final_mask += ext_flag[:, :] - else: - final_mask = external_flag.get_data()[:, :] - external_flag.close() - - return final_mask.astype(np.int16, copy=False) - - def _mask_to_file(self, input_mask, output_fullpath): - """Mask to File. - - Save the mask to a fits file. - - Parameters - ---------- - input_mask : numpy.ndarray - Mask to save - output_fullpath : str - Path of the output file - - Raises - ------ - ValueError - If input_mask is type ``None`` - ValueError - If output_fullpath is type ``None`` - - """ - if input_mask is None: - raise ValueError("input mask file path not provided") - if output_fullpath is None: - raise ValueError("output mask file path not provided") - - out = file_io.FITSCatalogue( - output_fullpath, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - hdu_no=0, - ) - out.save_as_fits(data=input_mask, image=True) - - if self._config["MD"]["make"]: - out.open() - out.add_header_card( - "MRATIO", - self._ratio, - "ratio missing_pixels/all_pixels", - ) - out.add_header_card( - "MFLAG", - self._config["MD"]["im_flagged"], - f'threshold value {self._config["MD"]["thresh_flag"]:.3}', - ) - - # Write WCS information to header - if self._wcs: - header_wcs = self._wcs.to_header() - for card in header_wcs: - out.add_header_card( - card, - header_wcs[card], - header_wcs.comments[card], - ) - out.close() - - def _get_temp_dir_path(self, temp_dir_path): - """Get Temporary Directory Path. - - Create the path and the directory for temporary files. - - Parameters - ---------- - temp_dir_path : str - Path to the temporary directory, a value of ``OUTPUT`` will include - the temporary files in the run directory - - Returns - ------- - str - Path to the temporary directory - - Raises - ------ - ValueError - If ``temp_dir_path`` is of type ``None`` - - """ - if temp_dir_path is None: - raise ValueError("Temporary directory path not provided") - - path = temp_dir_path.replace(" ", "") - - if path == "OUTPUT": - path = f"{self._output_dir}/temp" - - path += "/" - if not os.path.isdir(path): - mkdir(path) - - return path diff --git a/src/shapepipe/modules/mask_runner.py b/src/shapepipe/modules/mask_runner.py deleted file mode 100644 index 13fbf9b6c..000000000 --- a/src/shapepipe/modules/mask_runner.py +++ /dev/null @@ -1,120 +0,0 @@ -"""MASK RUNNER. - -Module runner for ``mask``. - -:Author: Axel Guinot, Martin Kilbinger - -""" - -from shapepipe.modules.mask_package.mask import Mask -from shapepipe.modules.module_decorator import module_runner - - -@module_runner( - version="1.0", - file_pattern=["image", "weight", "flag"], - file_ext=[".fits", ".fits", ".fits"], - depends=["numpy", "astropy"], - executes=["weightwatcher"], - numbering_scheme="_0", -) -def mask_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Mask Runner.""" - # Get number of input files - n_inputs = len(input_file_list) - - # Set options for 2 inputs - if n_inputs == 2: - ext_flag_name = None - ext_star_cat = None - - # Set options for 3 inputs - elif n_inputs == 3: - if config.getboolean(module_config_sec, "USE_EXT_FLAG"): - ext_flag_name = input_file_list[2] - ext_star_cat = None - elif config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = None - ext_star_cat = input_file_list[2] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag or " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Set options for 4 inputs - elif n_inputs == 4: - if config.getboolean( - module_config_sec, "USE_EXT_FLAG" - ) and config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = input_file_list[2] - ext_star_cat = input_file_list[3] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag and " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Raise error for invalid settings - else: - raise ValueError( - f'Found {n_inputs} inputs and these must be "image", "weight" and ' - + '"ext_flags", "ext_star_cat" (optional). Check the MASK_RUNNER ' - + "section of the config file to make sure you have the " - + "appropriate settings." - ) - - # Get path to mask configuration options - config_file = config.getexpanded(module_config_sec, "MASK_CONFIG_PATH") - - # Get mask HDU number - if config.has_option(module_config_sec, "HDU"): - hdu = config.getint(module_config_sec, "HDU") - else: - hdu = 0 - - # Get mask mask file name prefix - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - else: - prefix = "" - - outname_base = "flag" - - # Path to check for already created mask files - if config.has_option(module_config_sec, "CHECK_EXISTING_DIR"): - check_existing_dir = config.getexpanded( - module_config_sec, "CHECK_EXISTING_DIR" - ) - else: - check_existing_dir = None - - # Create instance of Mask - mask_inst = Mask( - *input_file_list[:2], - image_prefix=prefix.replace(" ", ""), - image_num=file_number_string, - config_filepath=config_file, - output_dir=run_dirs["output"], - path_external_flag=ext_flag_name, - outname_base=outname_base, - star_cat_path=ext_star_cat, - check_existing_dir=check_existing_dir, - hdu=hdu, - w_log=w_log, - ) - - # Process module - stdout, stderr = mask_inst.make_mask() - - # Return stdout and stderr - return stdout, stderr diff --git a/src/shapepipe/utilities/file_io.py b/src/shapepipe/utilities/file_io.py deleted file mode 100644 index 8a0964aa0..000000000 --- a/src/shapepipe/utilities/file_io.py +++ /dev/null @@ -1,45 +0,0 @@ -"""FILE I/O UTILITIES. - -Small, dependency-light helpers for publishing files the rest of the pipeline -treats as a cache. - -:Author: consolidated from workflow/scripts/star_cats.py and - scripts/python/create_star_cat.py - -""" - -import os - - -def write_atomic(table, path): - """Publish ``table`` at ``path`` all-or-nothing. - - Every caller's only cache test is existence (``Path.exists`` / - ``os.path.isfile``), so a write killed part-way -- job timeout, OOM, node - failure -- would otherwise leave a truncated FITS that every later run - trusts forever, and ``test -s`` passes on partial bytes. Writing to a temp - and renaming makes the visible file all-or-nothing: ``os.replace`` is atomic - within a directory. - - The temp keeps the target's suffix, because astropy picks its writer from - the extension. It is dot-prefixed and PID-tagged so it stays out of the - ``star_chunk-*`` / ``star_cat*`` globs the rules use, and two concurrent - writers cannot collide. - - Parameters - ---------- - table : astropy.table.Table - Table to write - path : str or pathlib.Path - Destination path - - """ - path = os.fspath(path) - directory, name = os.path.split(path) - tmp = os.path.join(directory or ".", f".tmp-{os.getpid()}-{name}") - try: - table.write(tmp, overwrite=True) - os.replace(tmp, path) - finally: - if os.path.exists(tmp): - os.remove(tmp) diff --git a/src/shapepipe/utilities/focal_plane.py b/src/shapepipe/utilities/focal_plane.py deleted file mode 100644 index 32770bd06..000000000 --- a/src/shapepipe/utilities/focal_plane.py +++ /dev/null @@ -1,92 +0,0 @@ -"""FOCAL PLANE GEOMETRY. - -The sky footprint of a MegaCam exposure, read from its image headers. Both -star-catalogue producers need exactly this and used to carry their own copy of -it -- ``workflow/scripts/star_cats.py`` (the HEALPix chunk store's ``cut``) and -``scripts/python/create_star_cat.py`` (the one-cone-per-exposure path the store -replaced). They must agree on which sky an exposure covers, so there is one -definition of it. - -:Author: consolidated from the two star-catalogue scripts - -""" - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.wcs import WCS - - -def get_wcs(header): - """Build the WCS by hand, from the linear terms only. - - Deliberately NOT ``WCS(header)``: it sidesteps distortion-convention - incompatibilities between these headers and astropy, and a footprint needs - nothing finer than the linear terms. - - Parameters - ---------- - header : astropy.io.fits.Header - Image header - - Returns - ------- - astropy.wcs.WCS - WCS object - - """ - final_wcs = WCS(naxis=2) - final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except KeyError: - final_wcs.wcs.cunit = ["deg", "deg"] - final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - final_wcs.wcs.cd = [ - [header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]], - ] - - return final_wcs - - -def ccd_center_and_radius(header): - """Return ``(ra_deg, dec_deg, radius_deg)`` for a single CCD. - - The radius is the half-diagonal: centre to the ``(0, 0)`` corner. - - """ - w = get_wcs(header) - (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( - [[header["NAXIS1"] / 2.0, header["NAXIS2"] / 2.0], [0, 0]], 1) - center = SkyCoord(ra_c * u.deg, dec_c * u.deg) - radius = center.separation(SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg - return float(ra_c), float(dec_c), float(radius) - - -def focal_plane_disc(image, n_ccd=40): - """Return ``(ra_deg, dec_deg, radius_deg)`` covering all CCDs of one exposure. - - The centre is the mean of the CCD centres; the radius is the largest - centre-to-CCD-centre distance plus that CCD's own half-diagonal. - - ONE ``fits.open`` for all the extensions: ``fits.getheader(image, ext)`` - opens the file, walks the HDU list to ``ext`` and closes again, so a loop - over it costs ``n_ccd`` opens and O(n^2) header seeks. - - """ - centers, radii = [], [] - with fits.open(image) as hdul: - for ext in range(1, n_ccd + 1): - ra_c, dec_c, radius = ccd_center_and_radius(hdul[ext].header) - centers.append((ra_c, dec_c)) - radii.append(radius) - - ras = np.array([c[0] for c in centers]) - decs = np.array([c[1] for c in centers]) - center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) - seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg - return (float(center.ra.deg), float(center.dec.deg), - float(np.max(seps + np.array(radii)))) diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py deleted file mode 100644 index 0bf077a78..000000000 --- a/src/shapepipe/utilities/vizier.py +++ /dev/null @@ -1,101 +0,0 @@ -"""VIZIER QUERY UTILITY. - -Consolidated Vizier query helper used by the mask module (to fetch the -reference star catalogue) and by ``scripts/python/create_star_cat.py``. -Retries over a list of mirror servers at progressively longer timeouts. - -:Author: Martin Kilbinger - -""" - -import random -import time - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astroquery.vizier import Vizier - - -VIZIER_SERVERS = [ - "vizier.cds.unistra.fr", - "vizier.cfa.harvard.edu", - "vizier.iucaa.in", -] - -VIZIER_TIMEOUTS = [10, 20, 40] - - -def query_vizier(ra, dec, radius_arcmin, cat_id): - """Query a Vizier catalogue with retries over timeouts and mirror servers. - - Parameters - ---------- - ra : float - Right ascension in degrees. - dec : float - Declination in degrees. - radius_arcmin : float - Cone-search radius in arcminutes. - cat_id : str - Vizier catalogue identifier (e.g. ``"I/305/out"`` for GSC 2.3). - - Returns - ------- - astropy.table.Table - First result table returned by Vizier. - - Raises - ------ - IndexError - If all server/timeout combinations return an empty result. - - """ - # Empirically, single-precision input positions can cause Vizier to return - # empty lists for some exposures; force double precision. - p = np.array([ra, dec], dtype="double") - coord = SkyCoord(ra=p[0] * u.deg, dec=p[1] * u.deg, frame="icrs") - - # Stagger concurrent queries to avoid hammering a single mirror. - time.sleep(random.uniform(0, 5)) - - for attempt, timeout in enumerate(VIZIER_TIMEOUTS): - for server in VIZIER_SERVERS: - v = Vizier( - row_limit=-1, timeout=timeout, vizier_server=server - ) - # cache=False: astroquery otherwise pickles every HTTP response into - # $HOME/.astropy/cache/astroquery/Vizier, ~2 MB per query. The - # workflow already caches the RESULT as a FITS catalogue on scratch - # and skips the query when it hits, so the pickle is pure duplicate — - # and at campaign scale (~25k exposures) it is ~50 GB against a - # 50 GB home quota. Home is for source and config, not for a second - # copy of the survey. - result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id, - cache=False, - ) - if len(result) > 0: - print( - f"Vizier query successful " - f"(server={server}, timeout={timeout}s)" - ) - return result[0] - print( - f"Vizier returned empty list at {coord}, " - f"{radius_arcmin:.2f} arcmin, " - f"server={server}, timeout={timeout}s" - ) - if attempt < len(VIZIER_TIMEOUTS) - 1: - wait = 10 * 2**attempt - print( - f"All servers failed, retrying in {wait}s " - f"(attempt {attempt + 1}/{len(VIZIER_TIMEOUTS)}, " - f"next timeout={VIZIER_TIMEOUTS[attempt + 1]}s)" - ) - time.sleep(wait) - - raise IndexError( - f"Vizier astroquery returned empty list at {coord}, " - f"radius={radius_arcmin} arcmin, catalog={cat_id}" - ) diff --git a/tests/module/test_collate_star_cat.py b/tests/module/test_collate_star_cat.py deleted file mode 100644 index 20d52ce23..000000000 --- a/tests/module/test_collate_star_cat.py +++ /dev/null @@ -1,70 +0,0 @@ -"""UNIT TESTS FOR STAR-CATALOGUE COLLATION PATHS. - -Pin the patch vs patch-less (v2.0) path and filename convention of -``scripts/python/collate_star_cat.py``. Runs up to v1.6 carry a ``P`` -token in both the input run directory and the output filename; v2.0 is -patch-less (``patch is None``) and drops that token, reading from a single -``/output`` root and writing ``validation_psf_conv-.fits`` — the -name still matched by the downstream ``validation_psf_conv-*`` glob. -""" - -import importlib.util -from pathlib import Path - -import pytest - -# The collation script lives under scripts/python (not an importable package), -# so load it by path. -_SCRIPT = ( - Path(__file__).resolve().parents[2] - / "scripts" - / "python" - / "collate_star_cat.py" -) -_spec = importlib.util.spec_from_file_location("collate_star_cat", _SCRIPT) -collate_star_cat = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(collate_star_cat) - - -@pytest.mark.parametrize( - "patch, exp_input, exp_output", - [ - ("3", "in/P3/output/", "out/P3"), - (None, "in/output/", "out"), - ], -) -def test_collate_paths(patch, exp_input, exp_output): - """v1.x carries the P token; v2.0 (patch None) drops it.""" - assert collate_star_cat.collate_paths("in", "out", patch) == ( - exp_input, - exp_output, - ) - - -@pytest.mark.parametrize( - "patch, expected", - [ - ("3", "validation_psf_conv-3-0.fits"), - (None, "validation_psf_conv-0.fits"), - ], -) -def test_output_filename(patch, expected): - """The patch token is present for v1.x and absent for v2.0.""" - assert collate_star_cat.output_filename("validation_psf", patch, 0) == expected - - -def test_output_filename_matches_downstream_glob(): - """Both layouts stay under the downstream ``validation_psf_conv-*`` glob.""" - for patch in ("1", None): - assert collate_star_cat.output_filename( - "validation_psf", patch, 5 - ).startswith("validation_psf_conv-") - - -@pytest.mark.parametrize("bad", ["v2", "2.0", "v1.7", ""]) -def test_invalid_version_raises(bad): - """A mistyped -V is rejected rather than falling through to v1.x.""" - obj = collate_star_cat.Convert() - obj._params["version_cat"] = bad - with pytest.raises(ValueError): - obj.run() diff --git a/tests/module/test_mask_ext.py b/tests/module/test_mask_ext.py deleted file mode 100644 index 6958c9e02..000000000 --- a/tests/module/test_mask_ext.py +++ /dev/null @@ -1,274 +0,0 @@ -"""UNIT TESTS FOR MODULE PACKAGE: MASK_EXT. - -Drives ``MaskExt`` against a synthetic healsparse map and a synthetic TAN WCS -to lock in the rasterization contract of ``mask_ext_runner``: the config-driven -bit->flag mapping (with bitwise-OR of multiple bits), chunk-seam independence, -RA wrap across 0/360, off-footprint (sentinel) handling, external-flag summing, -``int16`` output dtype, and WCS round-trip in the written FITS. - -The synthetic map is a high-resolution healsparse map covering only a tiny -patch on the sky; the synthetic WCS points an image at that patch so a subset -of pixels land on masked healpix cells and the rest fall off the footprint. -""" - -import numpy as np -import numpy.testing as npt -import pytest -from astropy.io import fits -from astropy.wcs import WCS - -healsparse = pytest.importorskip("healsparse") -hpgeom = pytest.importorskip("hpgeom") - -from shapepipe.modules.mask_ext_package.mask_ext import MaskExt - - -class _NullLogger: - def info(self, *_args, **_kwargs): - pass - - -# Sky patch the synthetic image and mask share. -CRVAL1 = 150.0 -CRVAL2 = 2.3 -NSIDE_SPARSE = 131072 # ~1.6 arcsec, matching the real UNIONS masks -NSIDE_COVERAGE = 32 -PIXSCALE_DEG = 0.187 / 3600.0 # UNIONS ~0.187"/px - - -def _make_wcs(naxis1, naxis2, crval1=CRVAL1, crval2=CRVAL2): - """Synthetic TAN WCS header centred on the shared patch.""" - w = WCS(naxis=2) - w.wcs.ctype = ["RA---TAN", "DEC--TAN"] - w.wcs.crval = [crval1, crval2] - w.wcs.crpix = [naxis1 / 2 + 0.5, naxis2 / 2 + 0.5] - w.wcs.cd = [[-PIXSCALE_DEG, 0.0], [0.0, PIXSCALE_DEG]] - return w - - -def _write_image(path, wcs_obj, naxis1, naxis2): - """Write a zero-valued image carrying the given WCS (defines pixel grid).""" - data = np.zeros((naxis2, naxis1), dtype=np.float32) - hdu = fits.PrimaryHDU(data) - # NAXIS* are set from the data shape; merge only the WCS keywords. - hdu.header.update(wcs_obj.to_header()) - hdu.writeto(path, overwrite=True) - - -def _make_map(masked_ra, masked_dec, bits, sentinel=0): - """Healsparse int32 map with the given (ra, dec) cells set to ``bits``.""" - hmap = healsparse.HealSparseMap.make_empty( - NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.int32, sentinel=sentinel - ) - pix = hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec) - bits = np.asarray(bits, dtype=np.int32) - # Several fine image pixels can share one healpix cell; keep the first bit - # value per unique cell (replace requires unique pixels). - pix, first = np.unique(pix, return_index=True) - hmap[pix] = bits[first] - return hmap - - -def _run(tmp_path, image_wcs, naxis1, naxis2, hmap, bit_flag_map, - off_map_flag=0, chunk_size=None, path_external_flag=None): - """Instantiate MaskExt on written synthetic inputs and rasterize/write.""" - image_path = str(tmp_path / "image.fits") - mask_path = str(tmp_path / "mask.hsp") - _write_image(image_path, image_wcs, naxis1, naxis2) - hmap.write(mask_path, clobber=True) - - inst = MaskExt( - image_path, - mask_path, - bit_flag_map, - image_num="-000-000", - output_dir=str(tmp_path), - w_log=_NullLogger(), - off_map_flag=off_map_flag, - path_external_flag=path_external_flag, - image_prefix="pipeline", - chunk_size=chunk_size, - ) - return inst - - -def test_parse_bit_flag_map(): - """String config parses into an int->int dict; malformed entries raise.""" - assert MaskExt.parse_bit_flag_map("64:1, 2048:2") == {64: 1, 2048: 2} - assert MaskExt.parse_bit_flag_map(" 64 : 1 ") == {64: 1} - with pytest.raises(ValueError): - MaskExt.parse_bit_flag_map("64") - with pytest.raises(ValueError): - MaskExt.parse_bit_flag_map("") - - -def test_bit_flag_mapping_and_off_map(tmp_path): - """Masked cells map their bit; unmasked (off-footprint) get off_map_flag. - - The image centre pixel lands on a cell carrying bit 64 -> flag 1; the map - covers only that centre cell, so every other pixel is off-footprint. - """ - naxis1 = naxis2 = 16 - w = _make_wcs(naxis1, naxis2) - # RA/Dec at the central pixel (0-based grid, origin=0). - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, off_map_flag=8) - flags = inst.rasterize() - - assert flags.dtype == np.int16 - assert flags.shape == (naxis2, naxis1) - # The centre cell is flagged 1; the rest of the image is off the footprint. - n_flagged_1 = np.sum(flags == 1) - assert n_flagged_1 >= 1 - assert np.all(flags[flags != 1] == 8) - assert np.sum(flags == 8) == flags.size - n_flagged_1 - - -def test_multi_bit_or(tmp_path): - """A cell carrying two bits gets the bitwise-OR of the mapped flags.""" - naxis1 = naxis2 = 8 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - # bit values 64 and 2048 set together on the centre cell. - hmap = _make_map([float(ra_c)], [float(dec_c)], [64 | 2048]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1, 2048: 2}) - flags = inst.rasterize() - # 1 | 2 == 3 on the masked cell. - assert 3 in np.unique(flags) - assert np.all(np.isin(np.unique(flags), [0, 3])) - - -def test_chunk_seam_independence(tmp_path): - """Result is independent of chunk size (no seam artifacts).""" - naxis1 = naxis2 = 40 - w = _make_wcs(naxis1, naxis2) - # Mask a band of cells spanning several image rows. - ys = np.arange(0, naxis2) - xs = np.full_like(ys, naxis1 // 2) - ra, dec = w.all_pix2world(xs, ys, 0) - hmap = _make_map(ra.astype(float), dec.astype(float), [64] * len(ys)) - - inst_full = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - full = inst_full.rasterize() - - for cs in (1, 3, 7, 40): - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}, chunk_size=cs) - npt.assert_array_equal(inst.rasterize(), full) - # Sanity: some pixels actually got flagged. - assert np.any(full == 1) - - -def test_ra_wrap(tmp_path): - """RA-wrap near 0/360: an image centred on RA~0 rasterizes correctly.""" - naxis1 = naxis2 = 16 - w = _make_wcs(naxis1, naxis2, crval1=0.0, crval2=2.3) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - # Straddling pixels produce RA both just below 360 and just above 0; the - # map cell is queried by its wrapped-into-[0,360) coordinate. - hmap = _make_map([float(np.mod(ra_c, 360.0))], [float(dec_c)], [64]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - flags = inst.rasterize() - # No crash, correct dtype, and the centre is flagged despite the wrap. - assert flags.dtype == np.int16 - assert np.any(flags == 1) - - -def test_external_flag_summing(tmp_path): - """External instrument flag image is summed into the rasterized mask.""" - naxis1 = naxis2 = 8 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) - - # External flag: a constant field of 16 (e.g. a saturation bit). - ext_path = str(tmp_path / "ext_flag.fits") - ext_data = np.full((naxis2, naxis1), 16, dtype=np.int16) - fits.PrimaryHDU(ext_data).writeto(ext_path, overwrite=True) - - inst = _run( - tmp_path, w, naxis1, naxis2, hmap, {64: 1}, - path_external_flag=ext_path, - ) - out_path = inst.make_mask() - - with fits.open(out_path) as hdul: - written = hdul[0].data - - # FITS stores big-endian; still a 2-byte signed int (int16). - assert written.dtype.kind == "i" and written.dtype.itemsize == 2 - # Every pixel gets +16 from the external flag; the centre cell also +1. - assert np.all(written >= 16) - assert 17 in np.unique(written) - - -def test_write_wcs_roundtrip(tmp_path): - """Written FITS carries the image WCS; pix2world round-trips.""" - naxis1 = naxis2 = 12 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_map([float(ra_c)], [float(dec_c)], [64]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - out_path = inst.make_mask() - - with fits.open(out_path) as hdul: - assert hdul[0].data.dtype.kind == "i" - assert hdul[0].data.dtype.itemsize == 2 - w_out = WCS(hdul[0].header) - - ra_in, dec_in = w.all_pix2world(cx, cy, 0) - ra_out, dec_out = w_out.all_pix2world(cx, cy, 0) - npt.assert_allclose([ra_in, dec_in], [ra_out, dec_out], rtol=0, atol=1e-9) - - -def _make_bool_map(masked_ra, masked_dec): - """Healsparse boolean map (True = masked) at the given (ra, dec) cells.""" - hmap = healsparse.HealSparseMap.make_empty( - NSIDE_COVERAGE, NSIDE_SPARSE, dtype=np.bool_, sentinel=False - ) - pix = np.unique(hpgeom.angle_to_pixel(NSIDE_SPARSE, masked_ra, masked_dec)) - hmap[pix] = np.ones(len(pix), dtype=np.bool_) - return hmap - - -def test_bool_map_flag_1(tmp_path): - """A boolean mask (True = masked) rasterizes through BIT_FLAG_MAP 1:1. - - This is the flavour of the real 2025 r-band UNIONS mask - (``mask_r_nside131072.hsp``): dtype bool, sentinel False, valid pixels - only where masked. - """ - naxis1 = naxis2 = 16 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {1: 1}) - flags = inst.rasterize() - - assert flags.dtype == np.int16 - assert np.sum(flags == 1) >= 1 - assert np.all(np.isin(np.unique(flags), [0, 1])) - - -def test_bool_map_wrong_bits_raise(tmp_path): - """Boolean mask + bits other than 1 would silently select nothing: raise.""" - naxis1 = naxis2 = 8 - w = _make_wcs(naxis1, naxis2) - cx, cy = naxis1 // 2, naxis2 // 2 - ra_c, dec_c = w.all_pix2world(cx, cy, 0) - hmap = _make_bool_map([float(ra_c)], [float(dec_c)]) - - inst = _run(tmp_path, w, naxis1, naxis2, hmap, {64: 1}) - with pytest.raises(ValueError, match="boolean healsparse map"): - inst.rasterize() diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini deleted file mode 100644 index d5b521080..000000000 --- a/workflow/config/cfis/config_exp_Ma.ini +++ /dev/null @@ -1,86 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Input file patterns: image, weight, external flag, external star catalogue -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -# True: the cat comes from $SP_RUN/star_cat_exp, the per-unit farm the -# exp_star_cat rule builds (40 per-CCD links to this exposure's one cat). -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -# Path to check for existing output mask files -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask deleted file mode 120000 index 9cd6a04b1..000000000 --- a/workflow/config/cfis/config_onthefly.mask +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/config_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask deleted file mode 120000 index 4743a87f0..000000000 --- a/workflow/config/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/config_tile_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/mask_default b/workflow/config/cfis/mask_default deleted file mode 120000 index 0970152ab..000000000 --- a/workflow/config/cfis/mask_default +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/mask_default \ No newline at end of file diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py deleted file mode 100644 index 2407af71b..000000000 --- a/workflow/scripts/star_cats.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -"""The campaign's GSC 2.3 star catalogue, as a HEALPix-chunked sky store. - -Masking needs, for every exposure, the bright stars over its focal plane. The -sky does not change between exposures, so the network cost of that is a property -of the campaign's SKY AREA, not of its exposure count: exposures overlap each -other ~7-10 deep, and a tile's exposures all look at the same square degree. - -So the store is chunked by sky, not by exposure. One GSC 2.3 cone query per -HEALPix pixel of NSIDE=32, written run-independently under the ``star_cats`` -config root and never fetched twice. A campaign that grows past the fetched -footprint queries only the chunks its new tiles add; one that grows within it -queries nothing. - -Two numbers set the scale. A full-UNIONS footprint is ~1.5k chunks against ~25k -exposures, so the QUERY COUNT drops ~16x. The queried AREA drops ~4x: the old -design covered the footprint ~8 times over (that is just the exposure overlap -depth), the new one ~2 times, the 2x being the price of bounding a HEALPix -quadrilateral by the cone Vizier speaks (see ``pixel_cone``) — 5.6-9 deg^2 for a -3.36 deg^2 pixel, ~40-60k rows and ~3-4 MB per chunk. - -Two subcommands, one module, deliberately: ``fetch`` and ``cut`` must agree -EXACTLY on which pixel holds which star, and a shared NSIDE constant in one file -is the only version of that agreement which cannot drift. - - fetch --tile-list ... --store ... --manifest ... - The campaign side. Turns the tile list into the set of pixels its - exposures can possibly need, fetches the missing ones, writes a manifest. - - cut --images ... --store ... --out ... - The per-exposure side, purely local: read the focal-plane footprint from - the exposure's image headers, load the chunks covering it, deduplicate, - and cut to the focal-plane disc. Reproduces byte-for-byte the same sky - selection the old one-query-per-exposure cone did. - -Geometry, and why the fetch pad is what it is. Chunk-need is computed from the -TILE list rather than from exposure pointings, because tile IDs are the one thing -known before any download: a pointing center means reading a FITS header of an -image get_images has not fetched yet, and the DAG needs the chunk set at parse -time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so -each tile is a disc of half-diagonal 0.354 deg; find_exposures gives a tile every -exposure whose footprint covers it, and the MegaCam focal plane is a disc of -radius 0.73 deg (measured on the cached catalogues). An exposure center is -therefore at most 0.354 + 0.73 deg from the tile center, and its stars 0.73 deg -beyond that: 1.81 deg, padded to ``PAD_DEG`` = 2.0. The pad is a perimeter cost -— negligible for a contiguous campaign, and paid once. - -The pad is a bound, not a promise: ``cut`` verifies that every chunk covering the -exposure it was handed is on disk, and fails loudly if one is not. A missing -chunk means the geometry above is wrong, and that must not degrade quietly into -an under-masked exposure. -""" - -import argparse -import json -import sys -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -import numpy as np -import healpy as hp -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.table import Table, vstack - -# The PYTHONPATH pin in profiles/nibi puts this checkout's src/ on the path (see -# exposure.smk's in_container), the same way the vizier helper is reached below. -from shapepipe.utilities.file_io import write_atomic -from shapepipe.utilities.focal_plane import focal_plane_disc - -# GSC 2.3. The same catalogue the mask module's own CDS path uses -# (mask.py: _CDS_cat_ID), so the store is a drop-in for it. -CAT_ID = "I/305/out" - -# NSIDE=32 -> 3.36 deg^2 per pixel, 12288 pixels over the sky. Chosen so one -# chunk is a couple of MegaCam focal planes: small enough that a Vizier query -# stays within a small multiple of the per-exposure queries this replaces, large -# enough that a full-UNIONS footprint is ~1.5k chunks rather than ~25k. The -# cone-vs-quadrilateral overhead is scale-free, so NSIDE trades query count -# against query size and nothing else. NESTED, so a chunk id is a hierarchical -# sky address and a future NSIDE change is a subdivision. -NSIDE = 32 -NEST = True - -# Angular padding on the disc used to select chunks (see the module docstring). -PAD_DEG = 2.0 - -# The MegaCam focal-plane disc, and the margin added to a pixel's own bounding -# cone. Both in degrees. -MARGIN_DEG = 0.02 - -# GSC 2.3's object id: the deduplication key where chunk cones overlap. -ID_COL = "GSC2.3" - - -# --- the chunk store -------------------------------------------------------- - - -def store_dir(store: Path) -> Path: - """Chunks live under the catalogue and resolution that produced them, so a - later NSIDE or catalogue change is a new directory beside the old one rather - than a silent reinterpretation of files already on disk.""" - return Path(store) / CAT_ID.replace("/", "_") / f"nside{NSIDE}" - - -def chunk_path(store: Path, ipix: int) -> Path: - return store_dir(store) / f"star_chunk-{ipix:06d}.fits" - - -def chunks_for_disc(ra_deg: float, dec_deg: float, radius_deg: float) -> list[int]: - """Every pixel that touches the disc, as sorted ids. - - ``inclusive=True`` makes this a conservative superset — the guarantee ``cut`` - relies on is that no star inside the disc lives in a pixel this omits. - """ - vec = hp.ang2vec(ra_deg, dec_deg, lonlat=True) - return sorted(int(i) for i in hp.query_disc( - NSIDE, vec, np.radians(radius_deg), inclusive=True, fact=4, nest=NEST)) - - -def pixel_cone(ipix: int) -> tuple[float, float, float]: - """(ra, dec, radius_arcmin) of a cone that CONTAINS pixel ``ipix``. - - Vizier speaks cones, HEALPix speaks quadrilaterals, so the query is the - pixel's bounding cone: its center, and the largest center-to-boundary - distance plus a margin. The cone spills over the pixel edges, which costs a - little duplication between neighbours and buys the containment ``cut`` - depends on. The duplicates are removed on read, by ``ID_COL``. - """ - ra_c, dec_c = hp.pix2ang(NSIDE, ipix, nest=NEST, lonlat=True) - ra_b, dec_b = hp.vec2ang(hp.boundaries(NSIDE, ipix, step=8, nest=NEST).T, - lonlat=True) - center = SkyCoord(ra_c * u.deg, dec_c * u.deg) - radius = center.separation(SkyCoord(ra_b * u.deg, dec_b * u.deg)).deg.max() - return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) - - -def read_chunks(store: Path, ipixels: list[int]) -> Table: - """Load and deduplicate the given chunks. - - A missing chunk is fatal (see the module docstring): it means the fetch - footprint did not cover this exposure, and an under-masked exposure is worse - than a failed job. - """ - missing = [i for i in ipixels if not chunk_path(store, i).exists()] - if missing: - raise SystemExit( - f"star chunk(s) {missing} not in {store_dir(store)}. The campaign's " - f"star_catalogue fetch did not cover this exposure — re-run it " - f"(and check that its tile list contains this exposure's tiles).") - - table = vstack([Table.read(chunk_path(store, i)) for i in ipixels], - metadata_conflicts="silent") - _, keep = np.unique(np.asarray(table[ID_COL]), return_index=True) - return table[np.sort(keep)] - - -# --- exposure footprint ----------------------------------------------------- -# The WCS construction and the focal-plane disc live in -# shapepipe.utilities.focal_plane, beside the vizier helper and imported the same -# way: create_star_cat.py needs exactly the same geometry, and the two must not -# be able to disagree about which sky an exposure covers. - - -def exposure_image(images_dir: Path) -> Path: - """The one multi-extension exposure image in a get_images output dir. - - That dir is a symlink farm holding ``image-.fitsfz`` plus its weight and - flag; only the image carries the 40 CCD WCSs. - """ - found = sorted(p for p in Path(images_dir).iterdir() if "image" in p.name) - if not found: - raise SystemExit(f"no image file in {images_dir}") - return found[0] - - -# --- fetch ------------------------------------------------------------------ - - -def campaign_chunks(tile_ids: list[str]) -> list[int]: - """Every chunk the campaign's exposures can need, from the tile list alone.""" - from shapepipe.utilities.cfis import get_tile_coord_from_nixy - - needed: set[int] = set() - for tile_id in tile_ids: - nix, niy = tile_id.split(".") - ra, dec = get_tile_coord_from_nixy(nix, niy) - needed.update(chunks_for_disc(ra.degree, dec.degree, PAD_DEG)) - return sorted(needed) - - -def fetch(args: argparse.Namespace) -> None: - from shapepipe.utilities.vizier import query_vizier - - tile_ids = [ln.strip() for ln in Path(args.tile_list).read_text().splitlines() - if ln.strip()] - needed = campaign_chunks(tile_ids) - out_dir = store_dir(args.store) - out_dir.mkdir(parents=True, exist_ok=True) - todo = [i for i in needed if not chunk_path(args.store, i).exists()] - print(f"star chunks: {len(needed)} needed for {len(tile_ids)} tiles, " - f"{len(todo)} to fetch -> {out_dir}", file=sys.stderr) - - def one(ipix: int) -> int: - ra, dec, radius_arcmin = pixel_cone(ipix) - table = query_vizier(ra, dec, radius_arcmin, CAT_ID) - write_atomic(table, chunk_path(args.store, ipix)) - print(f"chunk {ipix}: {len(table)} rows " - f"(ra={ra:.4f} dec={dec:.4f} r={radius_arcmin:.1f}')", - file=sys.stderr) - return len(table) - - # A handful of concurrent queries, never one per exposure: the same modest - # concurrency the per-exposure rule reached through --local-cores, now an - # explicit number instead of an accident of the head node's CPU count. - if todo: - with ThreadPoolExecutor(max_workers=args.workers) as pool: - list(pool.map(one, todo)) - - write_manifest(args, tile_ids, needed, len(todo)) - - -def write_manifest(args, tile_ids, needed, n_fetched) -> None: - """The rule's declared output. - - Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and - has no per-runner count floors, so there is nothing to compose and no - separate ``log:`` — under ``set -euo pipefail`` the job either completes or - aborts at the failing query, and snakemake's captured stderr is the evidence. - The manifest keeps the workflow's "one rule, one manifest" currency: written - last, and only when the content changed, so an unchanged campaign leaves the - mtime where it was rather than churning the `mtime` rerun-trigger. - """ - body = json.dumps({ - "stage": "star_catalogue", - "level": "campaign", - "status": "complete", - "catalogue": CAT_ID, - "nside": NSIDE, - "nest": NEST, - "pad_deg": PAD_DEG, - "store": str(store_dir(args.store)), - "n_tiles": len(tile_ids), - "n_chunks": len(needed), - "n_fetched": n_fetched, - "chunks": needed, - }, indent=2, sort_keys=True) - path = Path(args.manifest) - path.parent.mkdir(parents=True, exist_ok=True) - if not path.exists() or path.read_text() != body: - path.write_text(body) - - -# --- cut -------------------------------------------------------------------- - - -def cut(args: argparse.Namespace) -> None: - image = exposure_image(args.images) - ra, dec, radius = focal_plane_disc(image) - ipixels = chunks_for_disc(ra, dec, radius) - table = read_chunks(args.store, ipixels) - - center = SkyCoord(ra * u.deg, dec * u.deg) - stars = SkyCoord(np.asarray(table["RAJ2000"]) * u.deg, - np.asarray(table["DEJ2000"]) * u.deg) - inside = table[center.separation(stars).deg <= radius] - - print(f"{image.name}: ra={ra:.4f} dec={dec:.4f} r={radius:.4f} deg, " - f"{len(ipixels)} chunks -> {len(inside)} stars", file=sys.stderr) - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - write_atomic(inside, out) - - -# --- CLI -------------------------------------------------------------------- - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - sub = p.add_subparsers(dest="cmd", required=True) - - f = sub.add_parser("fetch", help="fetch the campaign footprint's chunks") - f.add_argument("--tile-list", required=True, type=Path) - f.add_argument("--store", required=True, type=Path) - f.add_argument("--manifest", required=True, type=Path) - f.add_argument("--workers", type=int, default=4) - f.set_defaults(func=fetch) - - c = sub.add_parser("cut", help="cut one exposure's catalogue from the store") - c.add_argument("--images", required=True, type=Path, - help="a get_images output dir holding image-.fitsfz") - c.add_argument("--store", required=True, type=Path) - c.add_argument("--out", required=True, type=Path) - c.set_defaults(func=cut) - - args = p.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() From 659168ad791c72b96867c6b54a5d6ceb02b3c0e4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:45:44 -0400 Subject: [PATCH 07/17] feat(mask_query): flag exposure detections against external healsparse masks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One healsparse lookup, two consumers. shapepipe.utilities.mask_query holds the primitive: `query_map` reads a map and returns its value at each (RA, Dec), and `flag_positions` ORs several maps into one integer per object. `make_cat` now calls query_map for its per-band MASK_ columns instead of opening healsparse itself, and the new `mask_query` module runs between sextractor and setools on exposures, writing an integer FLAG_EXT (0 = clean) into a NEW sexcat_ext.fits beside the input — the spread_model `add` pattern, so the LDAC_IMHEAD HDU survives and nothing mutates in place. star_selection.setools gains `FLAG_EXT == 0` beside every `IMAFLAGS_ISO == 0`: setools expressions have no bitwise operators, so the bit selection happens in the module (MASK_PATHS, optional MASK_BITS) and the config only tests for zero. The one interpretive choice is off-coverage. get_values_pos returns a map's sentinel outside its coverage — False for boolean maps, -1 for integer ones. make_cat passes that through verbatim (its documented off-map flag), but flag_positions treats it as NOT flagged for both kinds: OR-ing -1 in would flag every object a map does not reach, i.e. a map whose footprint stops short of an exposure would silently reject all of its stars. Off-coverage counts are logged. Tests build tiny nside_sparse=4096 maps and a three-HDU LDAC fixture and lock in the boolean/integer/MASK_BITS/OR/off-coverage cases, that the LDAC structure and the input file survive, and that make_cat's verbatim behaviour is unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- .../modules/make_cat_package/make_cat.py | 11 +- .../modules/mask_query_package/__init__.py | 51 +++++ .../modules/mask_query_package/mask_query.py | 94 ++++++++ src/shapepipe/modules/mask_query_runner.py | 73 +++++++ src/shapepipe/utilities/__init__.py | 2 +- src/shapepipe/utilities/mask_query.py | 152 +++++++++++++ tests/module/test_mask_query.py | 206 ++++++++++++++++++ 7 files changed, 583 insertions(+), 6 deletions(-) create mode 100644 src/shapepipe/modules/mask_query_package/__init__.py create mode 100644 src/shapepipe/modules/mask_query_package/mask_query.py create mode 100644 src/shapepipe/modules/mask_query_runner.py create mode 100644 src/shapepipe/utilities/mask_query.py create mode 100644 tests/module/test_mask_query.py diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index d6c38134d..946df5a8e 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -16,6 +16,7 @@ from sqlitedict import SqliteDict from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query def get_output_name(output_dir, file_number_string): @@ -246,6 +247,9 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log): returns the map's sentinel — ``-1`` for integer maps — verbatim), which is the documented off-map flag. + The lookup itself is ``shapepipe.utilities.mask_query.query_map``, + shared with the ``mask_query`` module: one primitive, two consumers. + Parameters ---------- final_cat_file : file_io.FITSCatalogue @@ -256,17 +260,14 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log): Logging instance """ - import healsparse - final_cat_file.open() ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"]) dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"]) for band, path in band_paths.items(): w_log.info(f"Query external mask for band {band}: {path}") - mask_map = healsparse.HealSparseMap.read(path) - values = mask_map.get_values_pos(ra, dec, lonlat=True) - final_cat_file.add_col(f"MASK_{band}", np.asarray(values)) + values = mask_query.query_map(path, ra, dec) + final_cat_file.add_col(f"MASK_{band}", values) final_cat_file.close() diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py new file mode 100644 index 000000000..b4b1d92c1 --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -0,0 +1,51 @@ +"""MASK QUERY PACKAGE. + +This package contains the module for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +:Parent module: ``sextractor_runner`` + +:Input: Single-exposure single-CCD SExtractor catalogue + +:Output: The same catalogue with an added ``FLAG_EXT`` column + +Description +=========== + +ShapePipe consumes sky-fixed masks by querying them, not by rasterizing them. +This module sits between SExtractor and ``setools`` on the exposure chain: it +reads each detection's windowed world position (``XWIN_WORLD``, ``YWIN_WORLD`` +in the ``LDAC_OBJECTS`` extension) and looks it up in the configured healsparse +maps, writing one integer column: + +``FLAG_EXT`` + ``0`` for an object no configured map flags, nonzero otherwise. The nonzero + value is the bitwise OR of the contributing map values, so it says *which* + bits fired, but nothing downstream is required to read it that way. + +The single column exists because ``setools`` expressions support only +``< > <= >= == !=`` — no bitwise operators — so the bit selection has to +happen here. ``star_selection.setools`` cuts on ``FLAG_EXT == 0`` beside its +existing ``IMAFLAGS_ISO == 0``: instrument flags reject pixels, the queried +masks reject objects. + +The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with +``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is +written once. That module's docstring documents the off-coverage convention. + +Module-specific config file entries +=================================== + +MASK_PATHS : str + Comma-separated healsparse map paths to query +MASK_BITS : int, optional + Bit mask applied to integer maps (``value & MASK_BITS`` flags); default is + to flag on any nonzero value. Ignored for boolean maps, which flag on + ``True`` +PREFIX : str, optional + Output file prefix + +""" + +__all__ = ["mask_query"] diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py new file mode 100644 index 000000000..16f3c5ff5 --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -0,0 +1,94 @@ +"""MASK QUERY. + +Class to flag SExtractor detections against external healsparse masks. + +:Author: Claude Fable 5, for PR #847 + +""" + +import numpy as np + +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + + +class MaskQuery(object): + """Mask Query. + + Query external healsparse masks at every detection of a SExtractor + catalogue and write the result as a single ``FLAG_EXT`` column into a copy + of that catalogue. + + Parameters + ---------- + sexcat_path : str + Path to the input SExtractor catalogue + output_path : str + Path to the output catalogue + mask_paths : list + Paths to the healsparse maps to query + bits : int, optional + Bit mask applied to integer maps; default ``None`` flags any nonzero + value + w_log : logging.Logger, optional + Logging instance + + """ + + def __init__( + self, + sexcat_path, + output_path, + mask_paths, + bits=None, + w_log=None, + ): + + self._sexcat_path = sexcat_path + self._output_path = output_path + self._mask_paths = mask_paths + self._bits = bits + self._w_log = w_log + + def process(self): + """Process. + + Query the masks and write the flagged catalogue. + + Returns + ------- + int + Number of flagged objects + + """ + ori_cat = file_io.FITSCatalogue( + self._sexcat_path, + SEx_catalogue=True, + ) + ori_cat.open() + data = ori_cat.get_data() + ra = np.copy(data["XWIN_WORLD"]) + dec = np.copy(data["YWIN_WORLD"]) + + flag = mask_query_util.flag_positions( + self._mask_paths, + ra, + dec, + bits=self._bits, + w_log=self._w_log, + ) + # int32 is what the catalogue carries; the UNIONS bit table needs 12 + # bits, and no OR of it can overflow. + flag = flag.astype(np.int32) + + new_cat = file_io.FITSCatalogue( + self._output_path, + SEx_catalogue=True, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + ori_cat.add_col( + "FLAG_EXT", flag, new_cat=True, new_cat_inst=new_cat + ) + ori_cat.close() + + return int(np.count_nonzero(flag)) diff --git a/src/shapepipe/modules/mask_query_runner.py b/src/shapepipe/modules/mask_query_runner.py new file mode 100644 index 000000000..fc2a6679a --- /dev/null +++ b/src/shapepipe/modules/mask_query_runner.py @@ -0,0 +1,73 @@ +"""MASK_QUERY RUNNER. + +Module runner for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +""" + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.modules.module_decorator import module_runner +from shapepipe.utilities import mask_query as mask_query_util + + +@module_runner( + version="1.0", + input_module="sextractor_runner", + file_pattern=["sexcat"], + file_ext=[".fits"], + depends=["numpy", "healsparse"], +) +def mask_query_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Mask Query Runner.""" + sexcat_path = input_file_list[0] + + # Get file prefix (optional) + if config.has_option(module_config_sec, "PREFIX"): + prefix = config.get(module_config_sec, "PREFIX") + if (prefix.lower() != "none") & (prefix != ""): + prefix = prefix + "_" + else: + prefix = "" + else: + prefix = "" + + mask_paths = mask_query_util.parse_map_paths( + config.getexpanded(module_config_sec, "MASK_PATHS") + ) + if not mask_paths: + raise ValueError( + f"[{module_config_sec}] MASK_PATHS is empty; the module has" + + " nothing to query." + ) + + # Any nonzero map value flags unless a bit selection is given + if config.has_option(module_config_sec, "MASK_BITS"): + bits = config.getint(module_config_sec, "MASK_BITS") + else: + bits = None + + output_path = ( + f'{run_dirs["output"]}/{prefix}sexcat_ext{file_number_string}.fits' + ) + + mq_inst = MaskQuery( + sexcat_path, + output_path, + mask_paths, + bits=bits, + w_log=w_log, + ) + n_flagged = mq_inst.process() + + w_log.info(f"FLAG_EXT nonzero for {n_flagged} objects") + + # No return objects + return None, None diff --git a/src/shapepipe/utilities/__init__.py b/src/shapepipe/utilities/__init__.py index 5f0eab2d6..42e234003 100644 --- a/src/shapepipe/utilities/__init__.py +++ b/src/shapepipe/utilities/__init__.py @@ -7,4 +7,4 @@ """ -__all__ = ["file_system", "cfis", "galaxy", "summary"] +__all__ = ["file_system", "cfis", "galaxy", "mask_query", "summary"] diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py new file mode 100644 index 000000000..0cc64255d --- /dev/null +++ b/src/shapepipe/utilities/mask_query.py @@ -0,0 +1,152 @@ +"""MASK QUERY. + +The one healsparse lookup in ShapePipe. + +ShapePipe does not generate or rasterize masks. Sky-fixed masks are supplied +as healsparse maps and are consumed by *querying them at object positions*: +every object gets its mask value(s) as catalogue columns, and rejection happens +at the catalogue level, never at the pixel level. The only mask that still +reaches pixels is the per-exposure instrument flag image delivered with it. + +Two callers share the primitive defined here: + +* ``make_cat`` writes one ``MASK_`` column per configured map, carrying + the map value verbatim (no interpretation, no filtering); +* ``mask_query`` writes a single integer ``FLAG_EXT`` column onto the exposure + SExtractor catalogue, combining the configured maps into "clean (0) or + flagged (nonzero)" so that ``setools`` — whose expression language has no + bitwise operators — can cut on ``FLAG_EXT == 0``. + +Coverage +-------- +``healsparse.HealSparseMap.get_values_pos`` returns a map's *sentinel* for +positions outside its coverage: ``False`` for boolean maps, and typically +``-1`` for integer maps. ``make_cat`` passes that sentinel through verbatim, +which is the documented off-map flag for the final catalogue. + +``flag_positions`` instead treats off-coverage as **not flagged**, for both map +kinds. This makes the integer case agree with the boolean case (whose sentinel +is literally ``False``) rather than diverge from it, and it keeps a map whose +coverage does not reach an exposure from silently rejecting every star on it. +Off-coverage counts are logged so the situation is visible rather than silent. + +:Author: Claude Fable 5, for PR #847 + +""" + +import numpy as np + + +def parse_map_paths(paths_str): + """Parse Map Paths. + + Parse a comma-separated list of healsparse map paths. + + Parameters + ---------- + paths_str : str + Comma-separated map paths, e.g. ``/a/star.hsp, /b/maximask.hsp`` + + Returns + ------- + list + Map paths, stripped of surrounding whitespace, empty entries dropped + + """ + return [path.strip() for path in paths_str.split(",") if path.strip()] + + +def query_map(path, ra, dec): + """Query Map. + + Read a healsparse map and return its value at each world position. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Map value at each position; positions outside the map's coverage carry + the map's sentinel value + + """ + import healsparse + + mask_map = healsparse.HealSparseMap.read(path) + + return np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + + +def flag_positions(paths, ra, dec, bits=None, w_log=None): + """Flag Positions. + + Combine one or more healsparse masks into a single per-object integer flag. + + Each map contributes at each position: + + * boolean map: ``1`` where the map is ``True``, ``0`` elsewhere; + * integer map: the map value, optionally restricted to ``bits`` + (``value & bits``); ``0`` where the value is zero or the position is + outside coverage. + + Contributions are combined with a bitwise OR, so the returned flag is zero + for a clean object and carries the union of the bits that fired otherwise. + + Parameters + ---------- + paths : list + Paths to the healsparse maps to query + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + bits : int, optional + Bit mask applied to integer maps; default ``None`` means any nonzero + value flags + w_log : logging.Logger, optional + Logging instance + + Returns + ------- + numpy.ndarray + Integer flag per object, ``0`` for a clean object + + """ + ra = np.asarray(ra) + dec = np.asarray(dec) + flag = np.zeros(ra.size, dtype=np.int64) + + for path in paths: + values = query_map(path, ra, dec) + + if values.dtype == bool: + contribution = values.astype(np.int64) + n_off = 0 + else: + integer = values.astype(np.int64) + # Off-coverage: the sentinel, negative by healsparse convention. + # Zeroed rather than OR-ed in, see this module's docstring. + off_coverage = integer < 0 + n_off = int(np.count_nonzero(off_coverage)) + integer = np.where(off_coverage, 0, integer) + if bits is not None: + integer &= bits + contribution = integer + + flag |= contribution + + if w_log is not None: + w_log.info( + f"Mask query {path}: " + f"{int(np.count_nonzero(contribution))}/{ra.size} objects " + f"flagged, {n_off} outside coverage" + ) + + return flag diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py new file mode 100644 index 000000000..053dffd1e --- /dev/null +++ b/tests/module/test_mask_query.py @@ -0,0 +1,206 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MASK_QUERY. + +Exercises the exposure-side half of the query-everything mask design (PR #847): +``mask_query`` reads each SExtractor detection's windowed world position out of +the ``LDAC_OBJECTS`` extension, looks it up in the configured healsparse maps, +and writes a single integer ``FLAG_EXT`` column into a NEW catalogue beside the +input. + +What is locked in here: (1) boolean maps flag with ``1``, (2) integer maps +contribute their value and ``MASK_BITS`` restricts which bits do, (3) the +off-coverage sentinel (``-1``) never flags — the one place this differs from +``make_cat``'s verbatim pass-through, argued in +:mod:`shapepipe.utilities.mask_query` — (4) several maps OR together, and +(5) the input catalogue is left untouched while the LDAC structure survives. +""" + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Detection world positions (RA, Dec in degrees). The last one sits far from +# every map's coverage, so it exercises the off-coverage path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +def _write_map(path, value, dtype=np.int16, n_covered=2): + """Build an integer map carrying ``value`` at the first ``n_covered``. + + Everything else reads the ``-1`` sentinel, i.e. off coverage. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=-1 + ) + if n_covered: + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.full(n_covered, value, dtype=dtype), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_bool_map(path, n_covered=2): + """Build a boolean map, ``True`` at the first ``n_covered``.""" + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, np.bool_ + ) + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.ones(n_covered, dtype=np.bool_), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_sexcat(path): + """Write a synthetic LDAC SExtractor catalogue of known positions. + + Written with astropy rather than ``FITSCatalogue.save_as_fits``, because + creating an LDAC catalogue through that API requires an existing LDAC file + to copy the ``LDAC_IMHEAD`` HDU from. The three-HDU layout below is what + SExtractor writes and what ``SEx_catalogue=True`` (``hdu_no=2``) indexes. + """ + imhead = fits.BinTableHDU.from_columns( + [ + fits.Column( + name="Field Header Card", format="1A", array=np.array(["x"]) + ) + ], + name="LDAC_IMHEAD", + ) + objects = fits.BinTableHDU.from_columns( + [ + fits.Column(name="NUMBER", format="J", array=np.arange(len(RA))), + fits.Column(name="XWIN_WORLD", format="D", array=RA), + fits.Column(name="YWIN_WORLD", format="D", array=DEC), + fits.Column( + name="IMAFLAGS_ISO", + format="J", + array=np.zeros(len(RA), dtype="i4"), + ), + ], + name="LDAC_OBJECTS", + ) + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto( + str(path), overwrite=True + ) + return str(path) + + +def _read(path): + cat = file_io.FITSCatalogue(str(path), SEx_catalogue=True) + cat.open() + data = cat.get_data() + flag = np.copy(data["FLAG_EXT"]) + names = set(data.dtype.names) + cat.close() + return flag, names + + +def test_parse_map_paths(): + """Comma-separated paths parse, whitespace-tolerant, empties dropped.""" + assert mask_query_util.parse_map_paths( + " /a/star.hsp, /b/maximask.hsp ,, " + ) == ["/a/star.hsp", "/b/maximask.hsp"] + + +def test_flag_positions_integer_map(tmp_path): + """An integer map contributes its value; off-coverage stays clean.""" + path = _write_map(tmp_path / "m.hsp", 4, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [4, 4, 4, 0] + ) + + +def test_flag_positions_bits_restrict(tmp_path): + """MASK_BITS selects which bits of an integer map flag.""" + path = _write_map(tmp_path / "m.hsp", 1028, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=4), [4, 4, 4, 0] + ) + # A bit the map does not carry leaves everything clean. + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=2), [0, 0, 0, 0] + ) + + +def test_flag_positions_ors_maps(tmp_path): + """Several maps combine with a bitwise OR.""" + a = _write_map(tmp_path / "a.hsp", 4, n_covered=1) + b = _write_map(tmp_path / "b.hsp", 1024, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([a, b], RA, DEC), [1028, 1024, 1024, 0] + ) + + +def test_flag_positions_boolean_map(tmp_path): + """A boolean map flags with 1; its False sentinel stays clean.""" + path = _write_bool_map(tmp_path / "bool.hsp", n_covered=2) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [1, 1, 0, 0] + ) + + +def test_mask_query_writes_flag_ext(tmp_path): + """The module writes FLAG_EXT into a new catalogue and counts the hits.""" + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-0.fits") + out_path = tmp_path / "sexcat_ext-000-0.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=_NullLogger() + ).process() + + assert n_flagged == 2 + flag, names = _read(out_path) + npt.assert_array_equal(flag, [4, 4, 0, 0]) + assert np.issubdtype(flag.dtype, np.integer) + # The columns SExtractor wrote survive alongside the new one. + assert {"NUMBER", "XWIN_WORLD", "YWIN_WORLD", "IMAFLAGS_ISO"} <= names + # The LDAC structure survives: setools and psfex read HDU 2 by index. + with fits.open(str(out_path)) as hdus: + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + + # The input is not mutated: this module publishes a new file. + with fits.open(in_path) as hdus: + assert "FLAG_EXT" not in hdus[2].data.dtype.names + + +def test_mask_query_bits_and_all_clean(tmp_path): + """MASK_BITS reaches the module, and a miss leaves every object clean.""" + map_path = _write_map(tmp_path / "m.hsp", 1024, n_covered=3) + in_path = _write_sexcat(tmp_path / "sexcat-000-1.fits") + out_path = tmp_path / "sexcat_ext-000-1.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], bits=4, w_log=_NullLogger() + ).process() + + assert n_flagged == 0 + flag, _ = _read(out_path) + npt.assert_array_equal(flag, [0, 0, 0, 0]) From 51f25a5ee1d586da690ef64294b9c1dc44c01ab0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:46:11 -0400 Subject: [PATCH 08/17] refactor(pipeline): rewire the chains around the query design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposures. SExtractor now reads the instrument flag straight from split_exp (`FILE_PATTERN = image, weight, flag` — split_exp already writes an unprefixed flag.fits per CCD), with the run_sp_exp_Ma INPUT_DIR entry gone; mask_query_runner sits between it and setools in both the example and the workflow config, and star_selection.setools cuts on FLAG_EXT == 0 beside every IMAFLAGS_ISO == 0. Same rewiring in config_exp_mccd.ini, which shares that setools file. sextractor_runner's declared input_module follows the real parent. Tiles. config_tile_Sx_nomask.ini becomes THE tile config: tiles have no instrument flag image and no mask module now, so FLAG_IMAGE = False with default_noimaflags.param is the only variant, in example/cfis and in the sims dir. random_cat still needs a pixel mask image, but nothing in the pipeline produces one — its second input is now declared external (the healsparse-native replacement is #797), which is a real capability gap and is flagged as one in the config, the runner and docs/source/random_cat.md rather than papered over. Snakemake. `exp_psf` now depends on `exp_split` directly; `star_catalogue`, `exp_star_cat`, `exp_mask`, `star_cat_cmd`, `in_container` and the STAR_CAT_* helpers are gone, along with `config["star_cats"]` and the two mid-chain localrules. The `exp_short` group goes too, and the docstring says why: it existed to fuse exp_split with exp_mask, and a group of one rule submits exactly the job the ungrouped rule submits. completeness.py trades its exp_mask stage (mask_runner 40/1) for a mask_query_runner row inside exp_psf, floor 2 rather than 40 because setools tolerates the same sparse-CCD attrition either side of it; run_report drops the two stages; clean_exposure stops reclaiming link farms that no longer exist. Verified in the container: all 30 module runners import, every example and workflow config's MODULE list resolves through get_module_runners, and the workflow parses and builds its DAG (`snakemake --lint`, `-n`). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_Rc.ini | 9 +- example/cfis/config_exp_mccd.ini | 29 +- example/cfis/config_exp_psfex.ini | 38 ++- example/cfis/config_tile_Sx.ini | 17 +- example/cfis/config_tile_Sx_nomask.ini | 114 -------- example/cfis/star_selection.setools | 13 + ..._tile_Sx_nomask.ini => config_tile_Sx.ini} | 5 + src/shapepipe/modules/random_cat_runner.py | 4 +- .../modules/sextractor_package/__init__.py | 4 +- src/shapepipe/modules/sextractor_runner.py | 2 +- workflow/Snakefile | 34 +-- workflow/config.yaml | 26 +- workflow/config/cfis/config_exp_psfex.ini | 38 ++- workflow/rules/exposure.smk | 261 ++---------------- workflow/rules/prepare.smk | 11 +- workflow/rules/tile.smk | 34 +-- workflow/scripts/clean_exposure.py | 20 +- workflow/scripts/completeness.py | 23 +- workflow/scripts/run_report.py | 2 +- 19 files changed, 200 insertions(+), 484 deletions(-) delete mode 100644 example/cfis/config_tile_Sx_nomask.ini rename example/cfis_image_sims/{config_tile_Sx_nomask.ini => config_tile_Sx.ini} (91%) diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini index 2fec0a0fc..5d04e8e37 100644 --- a/example/cfis/config_Rc.ini +++ b/example/cfis/config_Rc.ini @@ -53,10 +53,13 @@ TIMEOUT = 96:00:00 ## Module options [RANDOM_CAT_RUNNER] -#INPUT_DIR = last:get_images_runner, last:mask_runner -INPUT_DIR = last:get_images_runner, $SP_RUN/output/run_sp_combined_flag:mask_runner +# The mask image is now an EXTERNAL product: ShapePipe generates no tile +# masks. Point the second entry at a directory of tile mask images matching +# NUMBERING_SCHEME below (the healsparse-native replacement for this module is +# the survey-window work, issue #797). +INPUT_DIR = last:get_images_runner, $SP_CONFIG/tile_masks -FILE_PATTERN = CFIS_image, pipeline_flag +FILE_PATTERN = CFIS_image, mask NUMBERING_SCHEME = 000-000 diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 092df0417..8e9ffbd8f 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -19,7 +19,7 @@ RUN_NAME = run_sp_exp_SxSePsf [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, +MODULE = sextractor_runner, mask_query_runner, setools_runner, mccd_preprocessing_runner, mccd_fit_val_runner, merge_starcat_runner, mccd_plots_runner @@ -62,11 +62,11 @@ TIMEOUT = 96:00:00 # - $SP_RUN/output #INPUT_DIR = . -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_MODULE = split_exp_runner -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag NUMBERING_SCHEME = -0000000-0 @@ -127,16 +127,31 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_MODULE = last:sextractor_runner -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat_sexcat NUMBERING_SCHEME = -0000000-0 +# External healsparse masks queried at each detection's (RA, Dec); see the +# mask_query module docstring. star_selection.setools cuts on FLAG_EXT == 0. +MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp + +; MASK_BITS = 1028 + + +[SETOOLS_RUNNER] + +INPUT_MODULE = last:mask_query_runner + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 199050927..10566d5ed 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -1,5 +1,8 @@ # ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -20,7 +23,7 @@ RUN_NAME = run_sp_exp_SxSePsfPi [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner +MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI @@ -57,12 +60,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Input from two modules -#INPUT_DIR = last:split_exp_runner, run_sp_exp_Ma:mask_runner -INPUT_DIR = last:split_exp_runner, last:mask_runner +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = last:split_exp_runner -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag # Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on # the decorator's 4-entry FILE_EXT default (length check fails at startup) @@ -127,16 +129,34 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_DIR = last:sextractor_runner -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 +# External healsparse masks queried at each detection's (RA, Dec). Any map +# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which +# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. +MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps ignore it. +; MASK_BITS = 1028 + + +[SETOOLS_RUNNER] + +INPUT_DIR = last:mask_query_runner + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini index 12f158508..9859e4237 100644 --- a/example/cfis/config_tile_Sx.ini +++ b/example/cfis/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options @@ -55,11 +60,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner, run_sp_tile_Mh_exp:merge_headers_runner +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag, log_exp_headers +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers -FILE_EXT = .fits, .fits, .fits, .sqlite +FILE_EXT = .fits, .fits, .sqlite # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 @@ -69,14 +74,14 @@ EXEC_PATH = source-extractor # SExtractor configuration files DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param DOT_CONV_FILE = $SP_CONFIG/default.conv # Use input weight image if True WEIGHT_IMAGE = True # Use input flag image if True -FLAG_IMAGE = True +FLAG_IMAGE = False # Use input PSF file if True PSF_FILE = False @@ -97,7 +102,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND, SEGMENTATION +CHECKIMAGE = BACKGROUND # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/example/cfis/config_tile_Sx_nomask.ini b/example/cfis/config_tile_Sx_nomask.ini deleted file mode 100644 index 731a8d338..000000000 --- a/example/cfis/config_tile_Sx_nomask.ini +++ /dev/null @@ -1,114 +0,0 @@ -# ShapePipe configuration file for tile detection - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers - -FILE_EXT = .fits, .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = False - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index 8330a1eff..f35043d99 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -1,4 +1,13 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties +## +## Two independent mask cuts, and they come from different places: +## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), +## delivered with the exposure and read by SExtractor; +## FLAG_EXT == 0 the external healsparse masks, queried per detection by +## the mask_query module (which map bits reach FLAG_EXT is +## that module's MASK_PATHS / MASK_BITS config). +## SETools expressions have no bitwise operators, so mask_query does the bit +## selection and this file only tests for zero. [MASK:preselect] MAG_AUTO > 0 @@ -7,11 +16,13 @@ FWHM_IMAGE > 0.3 / 0.187 FWHM_IMAGE < 1.5 / 0.187 FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 NO_SAVE [MASK:flag] FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 NO_SAVE @@ -23,6 +34,7 @@ FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 [MASK:fwhm_mag_cut] FWHM_IMAGE > 0 @@ -30,6 +42,7 @@ FWHM_IMAGE < 40 MAG_AUTO < 35 FLAGS == 0 IMAFLAGS_ISO == 0 +FLAG_EXT == 0 NO_SAVE # Split the 'star_selection' sample into diff --git a/example/cfis_image_sims/config_tile_Sx_nomask.ini b/example/cfis_image_sims/config_tile_Sx.ini similarity index 91% rename from example/cfis_image_sims/config_tile_Sx_nomask.ini rename to example/cfis_image_sims/config_tile_Sx.ini index a5c12771c..78bcfb646 100644 --- a/example/cfis_image_sims/config_tile_Sx_nomask.ini +++ b/example/cfis_image_sims/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py index c85854cb5..f180c9f7e 100644 --- a/src/shapepipe/modules/random_cat_runner.py +++ b/src/shapepipe/modules/random_cat_runner.py @@ -12,7 +12,9 @@ @module_runner( version="1.1", - file_pattern=["image", "pipeline_flag"], + # The mask image is an external input: ShapePipe generates no tile masks + # (the healsparse-native replacement for this module is issue #797). + file_pattern=["image", "mask"], file_ext=[".fits", "fits"], depends=["astropy"], numbering_scheme="_0", diff --git a/src/shapepipe/modules/sextractor_package/__init__.py b/src/shapepipe/modules/sextractor_package/__init__.py index abb0b4572..40f40ca91 100644 --- a/src/shapepipe/modules/sextractor_package/__init__.py +++ b/src/shapepipe/modules/sextractor_package/__init__.py @@ -4,10 +4,10 @@ :Author: Axel Guinot -:Parent modules: ``mask_runner``, ``merge_headers_runner`` (the latter only +:Parent modules: ``split_exp_runner``, ``merge_headers_runner`` (the latter only when ``MAKE_POST_PROCESS`` is ``True``) -:Input: Single-exposure single-CCD image, weight and flag files +:Input: Single-exposure single-CCD image, weight and instrument flag files :Output: SExtractor output catalogue diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index c36a8596d..e45885b06 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -19,7 +19,7 @@ # first three entries only. @module_runner( version="1.0.1", - input_module=["mask_runner", "merge_headers_runner"], + input_module=["split_exp_runner", "merge_headers_runner"], file_pattern=["image", "weight", "flag", "log_exp_headers"], file_ext=[".fits", ".fits", ".fits", ".sqlite"], executes=["source-extractor"], diff --git a/workflow/Snakefile b/workflow/Snakefile index b925adc06..8a8a2a9bf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -82,10 +82,6 @@ RUN_DIR = Path(config["run_dir"]) # Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no # second path: one root, exactly the pre-D5 layout. PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) -# Run-independent root for the mask star catalogues: the HEALPix chunk store the -# star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it -# (config.yaml explains the placement). -STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the @@ -309,13 +305,6 @@ SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") -# Same argument for star_cats.py, which both star-cat rules call: their params -# otherwise fingerprint nothing but paths, so an edit to the chunking or the cut -# would never rerun them. ONE hash for both rules because it is one script — and -# that is also why fetch and cut live in one module (they must agree on which -# pixel holds which star). The hash does NOT key the store path — see -# config.yaml's star_cats block on clearing the store after a semantic change. -STAR_CAT_HASH = script_hash("star_cats.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -509,8 +498,7 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale pre-fix file with the bare id. There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at - the committed config dir, and ``star_cat_exp`` is a real per-unit directory - built by the ``exp_star_cat`` rule, not a symlink into a shared pool. + the committed config dir. Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -589,18 +577,16 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# The aggregation targets, clean_exposure, clean_tile, star_catalogue and -# exp_star_cat run in the head process. The two clean rules are seconds of rmtree -# and hang off `all`; exp_star_cat is seconds of local FITS work; all three would -# otherwise be ~20k (clean_exposure, exp_star_cat) or ~23k (clean_tile) sbatch -# submissions at DR6 scale for work shorter than the scheduling latency. -# star_catalogue is one job either way, and local keeps its CDS concurrency the -# explicit number its thread pool sets (see exposure.smk). +# The aggregation targets, clean_exposure and clean_tile run in the head +# process. Both clean rules are seconds of rmtree and hang off `all`; submitted +# they would be ~20k (clean_exposure) or ~23k (clean_tile) sbatch submissions at +# DR6 scale for work shorter than the scheduling latency. # -# star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out -# of any future `group:` label: a local job cannot be fused into a submitted group. -# The two clean rules are DAG leaves and have no such constraint. -localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat +# Both are DAG LEAVES, so neither constrains a `group:` label. (A mid-chain +# localrule would: a local job cannot be fused into a submitted group. The old +# star-catalogue rules were exactly that, and they are gone with the internal +# mask generation.) +localrules: all, prepare_all_tiles, clean_exposure, clean_tile rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index 8e4de8506..e3f0355aa 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -53,30 +53,6 @@ products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# The mask star-catalogue root — run-independent, shared by every campaign, and -# holding two things: -# /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 -# query per HEALPix chunk (~3.4 deg^2, ~25k rows), written by -# `star_catalogue` over the tile list's footprint and never fetched twice; -# /exp/star_cat-.fits the per-exposure cuts -# `exp_star_cat` makes from those chunks, with no network at all. -# Network therefore scales with SKY AREA, not exposure count: exposures overlap -# ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. -# -# On the PERSISTENT root: the sky store is a durable science product bought with -# ~1.5k catalogue-server queries at DR6 scale, and re-buying it after a scratch -# purge is the one cost in this workflow that cannot be paid with local compute. -# (It sat on scratch through smk-g3 only because def-mjhudson /project was then -# hard-full at 27/27 TiB.) -# -# THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to -# workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun -# both rules — the script's hash is a param on each — but a chunk already on disk -# is skipped and only re-cut. Clear the store by hand when the change must reach -# the data. Changing NSIDE or the catalogue ID is the exception: those name the -# directory, so a change there fetches into a new one beside the old. -star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache - # The run index, and — sharing its directory — missing.json and run_report.json. # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction @@ -142,7 +118,7 @@ clean_tiles: true # # READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its # exposures' stores. If you later retry that tile, those exposure chains are -# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# gone and will be REBUILT from scratch — get_images, split, psf, per # exposure. That is correct, and expensive. Ignore a tile when you have decided # it is dead, not while you are still debugging it. clean_ignore_tiles: [] diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 0af871d8e..08f45f99b 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -1,5 +1,8 @@ # ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -20,8 +23,7 @@ RUN_DATETIME = False [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - +MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI MODE = SMP @@ -57,11 +59,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Input from two modules -INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag # Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on # the decorator's 4-entry FILE_EXT default (length check fails at startup) @@ -126,16 +128,34 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 +# External healsparse masks queried at each detection's (RA, Dec). Any map +# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which +# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. +MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps ignore it. +; MASK_BITS = 1028 + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/mask_query_runner/output + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 9fe3e7b29..af60dadbe 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,17 +1,22 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -----> exp_mask -> exp_psf - -> exp_star_cat --/ - star_catalogue ---------------/ - -``star_catalogue`` is campaign-level, not per-exposure: one fetch of the whole -footprint's stars, which every exposure's ``exp_star_cat`` then cuts locally. + exp_get_images -> exp_split -> exp_psf Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a run log. There is no `prepare_exposures` aggregation target: these chains hang off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). +NO MASK RULE, and that is the design (PR #847). ShapePipe generates no masks. +The only mask that reaches pixels is the instrument flag image delivered with +the exposure, which ``exp_split`` splits per CCD alongside image and weight and +SExtractor reads directly. Sky-fixed masks are healsparse maps, queried once per +object: ``mask_query`` (inside exp_psf's config chain) writes ``FLAG_EXT`` onto +each CCD's SExtractor catalogue for setools' star cut, and ``make_cat`` writes +the per-band ``MASK_`` columns on the tile side. Neither needs a rule, a +star catalogue, or a network fetch — hence no ``star_catalogue`` / ``exp_star_cat`` +here, and no ``exp_mask``. + NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, not over one invocation — reclamation here is clean_exposure's job (S5), driven @@ -19,26 +24,21 @@ by the accumulating index. A temp() here would delete an exposure the moment this invocation's readers finished and cascade destructive reruns across spatial neighbours the next time a tile is appended. -GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — -one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under -the 15-minute floor Alliance policy asks us to bundle away. The composition -rules are in prepare.smk's docstring; this chain is linear too, so the group -asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. - -The two rules NOT in it are structural, not taste: - * exp_psf is heavy (16 GB, 4 h) and never fuses with a short rule; - * exp_get_images cannot join, because ``exp_star_cat`` — a LOCALRULE, and so - ungroupable — sits between it and exp_mask. Pulling get_images in would make - the group both a dependency and a dependent of exp_star_cat, i.e. a cycle. - Starting the group at exp_split leaves star_cat's inputs entirely upstream - of it, so the group has one clean external edge. -Different exposures share no DAG edge, so this is one group job per exposure. +NO GROUPING. The ``exp_short`` group existed to fuse exp_split and exp_mask — +two rules whose medians were 1:28 and 1:54, both well under the 15-minute floor +Alliance policy asks us to bundle away — into one sbatch per exposure. With +exp_mask gone there is nothing to fuse: a group of one rule submits exactly the +job the ungrouped rule submits, and the label would only obscure that. The +composition rules, should a second short rule ever appear here, are in +prepare.smk's docstring. exp_get_images stays separate for the same reason it +always did (a download, retried on its own), and exp_psf is heavy (16 GB, 4 h) +and never fuses with a short rule. NUMBER_LIST ($SP_UNIT_NUM, see unit_num in the Snakefile) is set only for exp_split, whose numbering scheme IS the exposure id; never for get_images / -exp_mask / exp_psf, whose per-CCD or download numbering would turn tolerated -per-CCD attrition into a whole-exposure hard failure. It is a property of the -committed configs (config_exp_Sp.ini alone carries the entry). +exp_psf, whose per-CCD or download numbering would turn tolerated per-CCD +attrition into a whole-exposure hard failure. It is a property of the committed +configs (config_exp_Sp.ini alone carries the entry). """ rule exp_get_images: @@ -58,198 +58,9 @@ rule exp_get_images: shell: sp_shell("exp_get_images", "config_exp_Gie.ini") -# --- mask star catalogues --------------------------------------------------- -# Two rules, and the split between them is the design: the NETWORK is a function -# of the campaign's sky area, the per-exposure catalogue is a local cut. -# -# `star_catalogue` fetches the footprint's GSC 2.3 stars once, one Vizier query -# per HEALPix chunk, into a run-independent chunk store under config `star_cats`. -# `exp_star_cat` then reads the chunks covering an exposure's focal plane and -# cuts them to it — no network at all. workflow/scripts/star_cats.py holds both -# halves, the geometry they must agree on, and the arithmetic that motivates the -# split; its docstring is the reference for chunking, padding and query counts. - -# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE -# pointing at a path that does not exist inside the image, so requests is pointed -# at the bundle explicitly (proven in the p3-batch1 bash precedent). -STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" - -# The rules run star_cats.py inside the container (healpy, astroquery, astropy) -# but call apptainer THEMSELVES rather than letting the SDM wrap them -# (`container: None` on both): the CA bundle above and the exposure rule's -# host-side farm loop both need the explicit exec. bin/sp has loaded the -# apptainer module. -# -# WHICH image and WHICH arguments are not this file's to decide, and hand-rolling -# them here was a real divergence: these were the only two rules that ignored a -# user's dev sandbox, because they read `config['container']` — the shared /project -# fallback — instead of the image the Snakefile resolved for everything else. -# `_image` is that resolution (sandbox -> cached SIF -> config), and the profile's -# own apptainer-args are the same string the SDM splices onto every other rule, -# PYTHONPATH pin for shapepipe.utilities.{vizier,cfis} included. -# container.profile_apptainer_args() exists precisely so this file can read them -# rather than restate them. Only the CA bundle is added on top, and only when the -# command touches the network. -def in_container(cmd, *, network=False): - args = list(_container.profile_apptainer_args()) - if not args: - # Silently falling back would run these two rules with no --cleanenv and - # no PYTHONPATH pin, i.e. against a different src/ than every other rule. - raise WorkflowError( - f"Could not read apptainer-args from {_container.PROFILE_FILE}; " - f"star_catalogue and exp_star_cat build their apptainer line from it.") - if network: - # The container's certifi bundle, one --env per variable (the profile's - # own PYTHONPATH entry uses the same one-assignment-per-flag form). - args += [a for k in ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", - "CURL_CA_BUNDLE") - for a in ("--env", f"{k}={STAR_CAT_CA}")] - return f"apptainer exec {' '.join(args)} '{_image}' {cmd}" - - -# The campaign's star catalogue: a first-class durable science product, keyed by -# sky rather than by run. Chunk-need is recomputed from the tile list on every -# run and only the missing chunks are fetched, so appending tiles costs exactly -# the chunks they add. -# -# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the -# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the -# per-exposure rule reached by accident through --local-cores, now an explicit -# number that does not scale with the head node's CPU count. -# -# `tile_list_hash` is what makes the incremental behaviour visible to the DAG. -# The tile list is parse-time config, not a rule input (and the profile drops the -# `input` rerun-trigger anyway), so appending tiles would otherwise leave this -# rule up to date against a footprint that has grown. Hashing the list into a -# param reruns it, and the rerun fetches only what is new. -STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" - - -rule star_catalogue: - output: - manifest = STAR_CAT_MANIFEST - # No `log:` — see write_manifest() in star_cats.py. - # `cmd` is a params value, so placeholders in it are not formatted (see - # unit_pre in the Snakefile). Hence the explicit manifest path. - params: - cmd = in_container( - f"python {SCRIPTS}/star_cats.py fetch" - f" --tile-list '{config['tile_list']}' --store '{STAR_CATS}'" - f" --manifest '{STAR_CAT_MANIFEST}'", network=True), - tile_list_hash = hashlib.md5( - Path(config["tile_list"]).read_bytes()).hexdigest()[:12], - script_hash = STAR_CAT_HASH - container: - None - threads: 4 - retries: 2 - resources: - mem_mb = 4000, - runtime = 720 - shell: - "set -euo pipefail\n{params.cmd}" - - -# The per-exposure catalogue and the 40 per-CCD symlinks the mask module's -# numbering scheme needs. Local: one header read for the focal-plane footprint, -# a load of the chunks covering it, a radial cut. -# -# A LOCALRULE, for the reason the Snakefile's localrules line gives. -# -# The per-unit farm is a REAL directory holding exactly this exposure's 40 -# numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR -# and the file handler INTERSECTS the numbers found across INPUT_DIRs, so a -# symlink to a shared whole-store pool contributes every other exposure's numbers -# and the intersection is empty ("numbers ... do not intersect", live). -# -# TWO declared outputs, and the second one is the point. -# -# The manifest keeps the "one rule, one manifest" currency of every other rule: -# written last, unique to this rule, a record of what the farm points at, and -# deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the -# chunk store at no network cost. -# -# But a manifest attests FOREVER, and the two things it attests to both live -# outside the unit's manifests/ dir: the cut catalogue on /scratch (60-day purge) -# and the farm itself. Either can vanish under a manifest that still says -# "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is -# declared too — one link stands for all 40, they are created by the same loop -# in the same instant, and declaring 40 buys nothing. Snakemake's existence test -# is os.path.exists, which FOLLOWS symlinks and is therefore False for a link -# whose target the purge removed. A purged cut or a deleted farm makes the rule -# out of date, it reruns, and it re-cuts or re-links as needed. -def star_cat_cmd(exp): - """The whole rule body, as bash — carried as a params value because it - contains literal ``{}`` (the manifest JSON); see unit_pre in the Snakefile.""" - cut_dir = f"{STAR_CATS}/exp" - cat = f"{cut_dir}/star_cat-{exp}.fits" - work = exp_dir(exp) - farm = f"{work}/star_cat_exp" - images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" - manifest = exp_manifest(exp, "exp_star_cat") - body = json.dumps({ - "stage": "exp_star_cat", "level": "exp", "unit": exp, - "status": "complete", "cat": cat, "link_dir": farm, "n_links": 40, - }, indent=2, sort_keys=True) - return "\n".join([ - "set -euo pipefail", - # LEGACY-SYMLINK HAZARD. Unit dirs built before this rule existed carry - # star_cat_exp as a SYMLINK into the old shared star-cat pool. `mkdir -p` - # is a no-op on an existing symlink-to-directory, so the 40-link loop - # below followed it and wrote this exposure's links INTO THE SHARED POOL - # (520 stray links found live). Replace the link — never `rm -rf` it, - # which would recurse into the pool, and never touch a real directory: - # a real farm is this rule's own output and `ln -sfn` refreshes it. - f"[ -L '{farm}' ] && rm -f '{farm}' || true", - f"mkdir -p '{cut_dir}' '{farm}' '{work}/manifests'", - in_container(f"python {SCRIPTS}/star_cats.py cut" - f" --images '{images}' --store '{STAR_CATS}'" - f" --out '{cat}'"), - f"test -s '{cat}'", - # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the - # one focal-plane catalogue (pattern from the p3-batch1 precedent). - f"for ccd in $(seq 0 39); do ln -sfn '{cat}' " - f"'{farm}/star_cat-{exp}-'\"$ccd\"'.fits'; done", - # Byte-stable, and written only after the links exist: an unconditional - # write would move the mtime, which is a rerun-trigger. - f"tmp='{manifest}.tmp'", - "cat > \"$tmp\" <<'SP_STAR_CAT_JSON'", - body, - "SP_STAR_CAT_JSON", - f"cmp -s \"$tmp\" '{manifest}' && rm -f \"$tmp\" || mv -f \"$tmp\" '{manifest}'", - ]) - - -rule exp_star_cat: - input: - rules.exp_get_images.output.manifest, - # The chunks this cut reads. star_cats.py fails loudly on a chunk that is - # missing anyway, but the edge is what makes the fetch happen first. - rules.star_catalogue.output.manifest - output: - manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", - # The sentinel: ccd-0 of the 40-link farm (see above). - link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" - # No `log:`, for the same reason as star_catalogue above. - params: - cmd = lambda wc: star_cat_cmd(wc.exp), - # star_cats.py is external to the shell string, so the `code` - # rerun-trigger does not see it — same reason SCRIPT_HASH exists. - script_hash = STAR_CAT_HASH - container: - None - threads: 1 - retries: 2 - resources: - mem_mb = 4000, - runtime = 10 - shell: - "{params.cmd}" - # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: - group: "exp_short" input: rules.exp_get_images.output.manifest output: @@ -266,33 +77,13 @@ rule exp_split: shell: sp_shell("exp_split", "config_exp_Sp.ini") -rule exp_mask: - group: "exp_short" - input: - # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs - # and this exposure's own star_cat_exp farm. - rules.exp_split.output.manifest, - rules.exp_star_cat.output.manifest - output: - manifest = f"{EXP_DIR}/manifests/exp_mask.json" - log: - f"{EXP_DIR}/logs/exp_mask.json" - params: - pre = lambda wc: unit_pre("exp_mask", wc.exp), - script_hash = SCRIPT_HASH - threads: 4 - resources: - mem_mb = lambda wc, attempt: 8000 * attempt, - runtime = 120 - shell: - sp_shell("exp_mask", "config_exp_Ma.ini") - -# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# SExtractor -> mask_query (FLAG_EXT) -> setools star selection -> PSFEx model +# -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's # :warn on psfex_interp_runner. rule exp_psf: input: - rules.exp_mask.output.manifest + rules.exp_split.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_psf.json" log: diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 07405407f..fdf5d726b 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -29,11 +29,12 @@ cached group resources and re-sets ``attempt`` on every member (jobs.py), and ``retries: 2`` still governs. A retry re-runs the whole group, which is safe because every rule ``rm -rf``s its own run dir at start. -Star catalogues for masking are NOT a prepare-phase concern and not pre-run -input: the compute DAG fetches the campaign footprint's stars once -(``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in -exposure.smk, into a run-independent store. The tile side has no star-cat node -because it has no mask rule yet — see tile.smk. +There is no masking node in this phase, or in any other: ShapePipe generates no +masks (PR #847). The instrument flag image ships with the exposure and is split +per CCD by ``exp_split``; the sky-fixed healsparse masks are queried per object +inside the ShapePipe configs (``mask_query`` on exposures, ``make_cat`` on +tiles). Nothing is fetched, staged or rasterized, so there is nothing to +prepare. """ # No NUMBER_LIST for get_images — a download stage has nothing on disk to diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 74d985e2a..981ac8d65 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -41,18 +41,14 @@ The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor run that the shape chain does not need co-scheduled, and folding it in would add its runtime to a sum that has no room. -Note there is no `tile_mask` rule: the committed config chain is the -"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask -run), and no tile-mask config was committed in the S2 sweep. Adding the masked -variant is a config + one rule, at the config selector the PRD describes. - -That rule also needs a tile-side analogue of ``exp_star_cat``: tile star cats key -on TILE id, so they are a separate cache namespace and a separate node, and the -earliest point it can run is after ``tile_uncompress`` (create_star_cat.py's -``-k tile`` mode reads the uncompressed tile image's primary header). The mask -config would then read a real per-unit ``$SP_RUN/star_cat_tiles`` directory, -built the same way and for the same reason (the file handler intersects numbers -across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). +There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe +generates no masks: tiles have no instrument flag image of their own, so +tile_detect runs SExtractor with FLAG_IMAGE = False against +default_noimaflags.param (config_tile_Sx.ini — what used to be the "sx_nomask" +variant, now the only one). Sky-fixed masks reach the tile as CATALOGUE columns +instead: ``tile_make_cat``'s make_cat queries the configured healsparse maps at +every object's (RA, Dec) and writes one ``MASK_`` column per band, which +is what downstream selections cut on. """ # --- the node-local tile root (the I/O + scratch fix) ---------------------- @@ -176,8 +172,8 @@ def tile_local(tile): runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in `tile_merge_cats`, which has no pre_run), so the staging is attempted nine times per tile, eight of them concurrent siblings in one toposort level. It - is copy-to-a-temp-name plus `mv -f` -- the same all-or-nothing publish - shapepipe.utilities.file_io.write_atomic argues -- and it is UNCONDITIONAL, + is copy-to-a-temp-name plus `mv -f` -- an all-or-nothing publish, so a + reader never sees a partial file -- and it is UNCONDITIONAL, no `cp -u` and no already-there test. An interrupted `cp` leaves a truncated destination whose mtime is NEWER than the source, so `-u` would skip it forever, and nothing downstream would catch it: TILE_VIGNET_FRESH and @@ -390,9 +386,8 @@ def exp_manifests(wc, stage): return [ancient(p) for p in paths] def tile_exp_split(wc): return exp_manifests(wc, "exp_split") -def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") -def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_psf(wc) # Build the per-tile symlink forest. Declaring the exposure manifests as input @@ -873,10 +868,9 @@ rule tile_make_cat: # safe because nothing upstream of it survives, and tile_detect.json in # particular must never join the list. # -# A LOCALRULE (declared in the Snakefile), same as clean_exposure. Unlike -# exp_star_cat it is a DAG LEAF, so being local can never make it both a -# dependency and a dependent of a group — no `group:` label here, and none -# possible. +# A LOCALRULE (declared in the Snakefile), same as clean_exposure. It is a DAG +# LEAF, so being local can never make it both a dependency and a dependent of a +# group — no `group:` label here, and none possible. # # WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to # tile_local() mid-campaign reruns finished tiles unsatisfiably because their diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 728f6c527..b763a5573 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,17 +7,12 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its -star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are -reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted -here like every other, so the exposure's chain must read as unbuilt, and 40 -symlinks left behind are a farm no rule now owns. The catalogue itself lives in -the run-independent cache, so rebuilding the farm costs a relink and no query. +run_sp_exp_Gie/Sp/SxSePsfPi), its ``manifests/`` and its ``logs/``. That is the +entire exposure store: since PR #847 removed ShapePipe's mask generation there +is no run_sp_exp_Ma tree and no star-catalogue link farm to reclaim beside it. Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not -``rmtree``d. Legacy unit dirs carry ``star_cat_exp`` as a link into the old -shared pool, and an rmtree would recurse through it and delete the shared cache -for every other exposure in the campaign. +``rmtree``d, so a link into a shared store can never be recursed through. Deleting the manifests is deliberate and load-bearing, not tidiness: @@ -98,10 +93,9 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - # is_symlink() first, and OR'd with exists(): exists() follows the link, so a - # dangling legacy star_cat_exp would otherwise be skipped and survive. - candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs", - args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") + # is_symlink() first, and OR'd with exists(): exists() follows the link, so + # a dangling link would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs") targets = [t for t in candidates if t.is_symlink() or t.exists()] # Tombstone first, complete — then delete (see the module docstring). diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index cf67363db..0dfac0ebb 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -14,8 +14,8 @@ ``shapepipe_run`` failed:: rc=0 - shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} --log {log} --job-rc "$rc" || rc=1 + shapepipe_run -c $SP_CONFIG/config_exp_Sp.ini -b {threads} || rc=$? + completeness.py check exp_split {output} --log {log} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory @@ -82,12 +82,18 @@ # --- exposure chain --- "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, - "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree # AND the bash baseline (both 120/exposure). + # + # mask_query is one sexcat_ext per CCD — the count the deleted exp_mask + # stage used to carry, now inside this chain because querying a healsparse + # map at ~2k detections needs no rule of its own. Its floor tracks the + # runners either side of it: setools tolerates sparse-CCD attrition, so a + # hard 40 here would fail exposures the chain is designed to survive. "exp_psf": { "sextractor_runner": dict(expect=120, floor=2), + "mask_query_runner": dict(expect=40, floor=2), "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), "psfex_runner": dict(expect=80, floor=2), "psfex_interp_runner": dict(expect=40, floor=0, warn=True), @@ -163,18 +169,17 @@ def check_floor(stage, run_dir): # var its config does, so chunk K's check looks at chunk K's dir. # # EVERY ENTRY HERE (and in COMPLETENESS above) HAS A RULE. The table used to -# carry two stages that did not: `tile_mask` (run_sp_tile_Ma, mask_runner 1/1) -# and `tile_detect_uc` (run_sp_tile_Uc). The committed config chain is the -# "sx_nomask" tile_detect variant and no tile-mask config was committed, so both -# were unreachable — tile.smk's docstring is where the masked variant is argued, -# and it is a config plus a rule plus these two rows, added back together. +# carry stages that did not — `tile_mask` (run_sp_tile_Ma) and `tile_detect_uc` +# (run_sp_tile_Uc) — and, until PR #847, an `exp_mask` stage that did. ShapePipe +# now generates no masks at all: the sky-fixed healsparse maps are queried per +# object inside the exp_psf and tile_make_cat chains, so masking has no stage +# of its own on either side and is not coming back. STAGE_DIR = { "tile_get_images": ("tile", "run_sp_tile_Git"), "tile_uncompress": ("tile", "run_sp_tile_Uz"), "tile_find_exposures": ("tile", "run_sp_tile_Fe"), "exp_get_images": ("exp", "run_sp_exp_Gie"), "exp_split": ("exp", "run_sp_exp_Sp"), - "exp_mask": ("exp", "run_sp_exp_Ma"), "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), "tile_detect": ("tile", "run_sp_tile_Sx"), diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index f559e3276..bd117c09c 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -59,7 +59,7 @@ TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", "tile_merge_headers", "tile_detect", "tile_vignets", "tile_ngmix", "tile_merge_cats", "tile_make_cat"] -EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] # The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names # the mechanism that owns each). Their presence is therefore NOT evidence that a From 1bceb2da112ea4fa54cc480742659eb7e026b835 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:46:28 -0400 Subject: [PATCH 09/17] docs(mask): describe the query design, drop the deps only masking used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipeline_tutorial.md's "Mask images" section becomes "Masks" and explains the design instead of the old two-run mask procedure: healsparse maps queried per object into FLAG_EXT and MASK_, the instrument flag as the one mask that reaches pixels, no internet access and no star-catalogue download anywhere. pipeline_canfar.md loses its mask-tiles and mask-exposures steps and the combine_runs flag_* staging; random_cat.md gains a warning that its input mask images now come from outside ShapePipe; workflow/README.md and the sims README describe the chains as they now are. Dependencies. weightwatcher leaves the Dockerfile and the three docs pages that listed it — the deleted mask module was its only caller (`ww` appears nowhere else). astroquery and hpgeom leave pyproject.toml: astroquery had exactly two importers, utilities/vizier.py and star_cats.py, and hpgeom was the mask_ext rasterizer's. `uv lock` regenerated the manifest (astroquery, html5lib, pyvo removed; hpgeom stays, healsparse pulls it in). canfar_avail_results loses its -m / pipeline_flag check mode, and scripts/README.rst its create_star_cat entry. Left deliberately: shapepipe.utilities.summary{,_params_pre_v2}'s mask_runner entries. Those describe the pre-v2 CANFAR job map and parse the logs of runs that already exist on disk — removing them would break summary_run against historical trees without making anything current cleaner. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- Dockerfile | 4 +- docs/source/container.md | 6 +- docs/source/dependencies.md | 1 - docs/source/installation.md | 4 +- docs/source/pipeline_canfar.md | 48 +++---------- docs/source/pipeline_tutorial.md | 59 ++++++++-------- docs/source/random_cat.md | 67 ++++++------------- example/cfis_image_sims/README.md | 8 ++- .../config_tile_PiViVi_canfar_sx.ini | 2 +- pyproject.toml | 2 - scripts/README.rst | 9 --- scripts/python/canfar_avail_results.py | 14 ---- uv.lock | 48 +------------ workflow/README.md | 30 +++++---- 14 files changed, 92 insertions(+), 210 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0a549ae8f..f6d323706 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV SHELL=/bin/bash \ COVERAGE_FILE=/tmp/.coverage # System dependencies — three categories: -# - astromatic binaries (psfex, source-extractor, weightwatcher) ship as +# - astromatic binaries (psfex, source-extractor) ship as # Debian packages on bookworm; preferred over building from source. # - compilers and dev libs needed to build the heavier wheels (galsim, # mpi4py, python-pysap, fitsio). @@ -55,7 +55,7 @@ RUN apt-get update -y --quiet && \ libcfitsio-dev \ libproj-dev proj-bin \ libgl1-mesa-glx \ - psfex source-extractor weightwatcher && \ + psfex source-extractor && \ apt-get clean && rm -rf /var/lib/apt/lists/* # OpenMPI from source — required for hybrid Apptainer MPI on HPC clusters. diff --git a/docs/source/container.md b/docs/source/container.md index 174cde9dc..1a83b0f55 100644 --- a/docs/source/container.md +++ b/docs/source/container.md @@ -165,7 +165,7 @@ The Dockerfile does **not** duplicate Python deps — those come from The asymmetry is deliberate: Python deps go through pyproject + lockfile (reproducible, auditable), system deps go through Dockerfile (Debian's versioning). Don't `apt install` something that has a Python wheel; don't -`pip install` something Debian packages directly (e.g. `weightwatcher`). +`pip install` something Debian packages directly (e.g. `source-extractor`). ## Why this shape @@ -175,8 +175,8 @@ versioning). Don't `apt install` something that has a Python wheel; don't - **`uv sync --frozen`** at build time means the image is bit-exactly reproducible from a tagged commit, and impossible to ship with a stale lockfile. -- **Astromatic binaries from Debian** (`psfex`, `source-extractor`, - `weightwatcher`) instead of source builds — Debian carries the +- **Astromatic binaries from Debian** (`psfex`, `source-extractor`) + instead of source builds — Debian carries the GCC-compatibility patches that the previous Dockerfile had to apply inline with `sed`. - **Two targets** so canfar batch deployments stay slim while interactive diff --git a/docs/source/dependencies.md b/docs/source/dependencies.md index 9378bfe00..b907a452d 100644 --- a/docs/source/dependencies.md +++ b/docs/source/dependencies.md @@ -57,7 +57,6 @@ packages (no source builds), plus the MPI stack: |---------|------------| | [Source Extractor](https://www.astromatic.net/software/sextractor/) | {cite:p}`bertin:96` | | [PSFEx](https://www.astromatic.net/software/psfex/) | {cite:p}`bertin:11` | -| [WeightWatcher](https://www.astromatic.net/software/weightwatcher/) | {cite:p}`marmo:08` | | OpenMPI (5.0.x) | | Python dependencies themselves are managed with [uv](https://docs.astral.sh/uv/); diff --git a/docs/source/installation.md b/docs/source/installation.md index 83df0b6bb..ee3f8e182 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -38,8 +38,8 @@ docker pull ghcr.io/cosmostat/shapepipe:develop-runtime We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance. ``` -The image bundles the astromatic binaries (`source-extractor`, `psfex`, -`weightwatcher`), MPI (`mpi4py` + OpenMPI), and every Python dependency, so +The image bundles the astromatic binaries (`source-extractor`, `psfex`), +MPI (`mpi4py` + OpenMPI), and every Python dependency, so there is nothing else to install or build. To process data on a cluster with MPI, run the pipeline through Apptainer the same way you would any MPI job. diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 96add5758..926d36578 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -170,22 +170,16 @@ The downloaded tile weights are compressed. The following call uncompresses all. shapepipe_run -c cfis/config_tile_Uz.ini ``` -### Mask tiles +### Masks -This step is done globally for all tiles. There might be job failures or interruptions. The following -command to the `ShapePipe` job script can be run repeatedly; already created masks will be skipped. - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 4 -``` - -If masks were created in more than one run, i.e. situated in more than one output directory, these have to be -combined for subsequent pipeline module runs. This is done by creating a new output directory with symbolic -links, using the script - -```bash -combine_runs.bash -c flag_tile -``` +There is no masking step. `ShapePipe` generates no masks: the sky-fixed +healsparse maps are queried once per object, by `mask_query` on the exposure +catalogues (`FLAG_EXT`, cut by `setools`) and by `make_cat` on the final tile +catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` +config entries at the maps and nothing else is needed — no star-catalogue +download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that +touches pixels is the instrument flag image shipped with each exposure, which +`split_exp` splits per CCD. ## Tile detection @@ -205,7 +199,7 @@ canfar_submit_job -j 16 -f tile_numbers.txt -P N_PAR -v -J JMAX ### Exposure Processing -#### Option 0: Global split and exp masks (deprecated; used for earlier v1.x patch runs) +#### Option 0: Global split (deprecated; used for earlier v1.x patch runs) For this option, set `sp_local=0`. @@ -217,21 +211,7 @@ For `sp_local=-` both `mh_local` (0, 1) are ok: export mh_local=0 ``` -#### Option 0: Mask exposures (deprecated) - -Run repeatedly if necessary: - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 8 -``` - -Combine all runs: - -```bash -combine_runs.bash -c flag_exp -``` - -### Option 1: Local split and mask exposures (recommended) +### Option 1: Local split exposures (recommended) Optional: Enable flags for local split processing and merge header runs as @@ -258,12 +238,6 @@ First, determine the number of maximum jobs with the option `-s` (see above). Th canfar_submit_job -j 2 -v -f exp_shdu.txt -v -P N_PAR -J JMAX ``` -### Mask exposures - -```bash -canfar_submit_job -j 8 -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - ### Exposure detection ```bash diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 95a0aa94a..36dde0ccd 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -44,11 +44,11 @@ Naming and numbering of the input files can closely follow the original image na A stacked image is also called *tile*. These files are used on input by `ShapePipe`. The pixel data can contain the observed image, a weight map, or a flag map. Tile images and weights are created in the case of CFIS by Stephen Gwyn using a combination of `swarp` and his own software. Examples of file names are - `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tile flag files - are created the mask module of `ShapePipe` (see [Mask images](#mask-images)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for + `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tiles have no flag file + (see [Masks](#masks)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for clarity, we include the string `image` for a tile image type. Default convention: **-.fits** - Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz`, `pipeline_flag-239-293.fits` + Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz` - Database catalogue files For very large files that combine information from multiple tiles or single exposures, `ShapePipe` creates `sqlite` @@ -128,8 +128,6 @@ for all options. This script creates the subdirectory `$SP_RUN/output` to store all pipeline outputs (log files, diagnostics, statistics, output images, catalogues, single-exposure headers with WCS information). -Optionally, the subdir `output_star_cat` is created by the used to store the external star catalogues for masking. This is only necessary if the pipeline is run on a cluster without internet connection to access star catalogues. In that case, the star catalogues need to be retrieved outside the pipeline, for example on a login node, and copied to `output_star_cat`. - The job script automaticall performs a number of subsequent calls to the `ShapePipe` executable `shapepipe_run`, as ```bash shapepipe_run -c $SP_CONFIG/.ini @@ -189,32 +187,31 @@ Finally, the headers of all single-exposure single-CCD files are merged into a s Two output directories are created, `run_sp_Uz` for `uncompress_fits`, and `run_sp_exp_SpMh` for the output of the modules `split_exp` (`Sp`) and `merge_headers` (`Mh`). -## Mask images - -Run -```bash -job_sp TILE_ID -j 4 -``` -to mask tile and single-exposure single-CCD images. Both tasks are performed by two calls to the `mask` runner. - -Note that internet access is required for this step, since a reference star catalogue is downloaded. - -The output of both masking runs are stored in the output directory `run_sp_MaMa`, with run 1 (2) of -`mask` corresponding to tiles (exposures). - -**Diagnostics:** Open a single-exposure single-CCD image and the corresponding pipeline flag -in `ds9`, and display both frames next to each other. Example -```bash -ds9 image-2113737-10.fits pipeline_flag-2113737-10.fits -``` -Choose `zoom fit` for both frames, click `scale zscale` for the image, and `color aips0` for the flag, to display something like this: - - - -By eye the correspondence between the different flag types and the image can be -seen. Note that the two frames might not match perfectly, since (a) WCS -information is not available in the flag file FITS headers; (b) the image can -have a zero-padded pixel border, which is not accounted for by `ds9`. +## Masks + +`ShapePipe` does not generate masks. Sky-fixed masks — star halos, stars, +manual masks for large galaxies, per-band coverage, MaxiMask defects — are +supplied as [healsparse](https://healsparse.readthedocs.io) maps and are +consumed by *querying them at object positions*, never by rasterizing them onto +pixels. Two modules do the querying, from the same shared lookup +(`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and +`setools` on the single-exposure single-CCD catalogues and writes one integer +`FLAG_EXT` column (0 = clean), which `star_selection.setools` cuts on so that +masked objects never enter the PSF star sample; `make_cat` writes one +`MASK_` column per band onto the final tile catalogue, carrying the map +value verbatim so downstream selections choose their own cuts. Map paths and +bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and +`MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. + +No internet access is needed at any point, and there is no reference star +catalogue to download. + +The one mask that still reaches pixels is the **instrument flag image** +(`p.flag.fits.fz`) delivered with each exposure, which records bad columns +and saturation. `split_exp` splits it per CCD beside the image and weight, +`sextractor` reads it as `IMAFLAGS_ISO`, and `ngmix` zero-weights flagged +pixels in its postage stamps. Tiles have no such image, so tile detection runs +with `FLAG_IMAGE = False`. ## Detect objects on tiles and process stars on single exposures diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md index a930d40ed..b5ae9931d 100644 --- a/docs/source/random_cat.md +++ b/docs/source/random_cat.md @@ -3,16 +3,25 @@ This section describes how to create tile-based random catalogues and healpix masks, and combined randoms and masks for a selection of tiles. -The masked regions are obtained on input from ShapePipe pixel mask ("pipeline flag") -files. +The masked regions are obtained on input from per-tile pixel mask images. + +```{warning} +**ShapePipe no longer produces those images.** The pipeline generates no masks +at all: sky-fixed masks are healsparse maps, queried once per object into +catalogue columns (see [Masks](pipeline_tutorial.md#masks)), and tiles have no +flag image. `random_cat_runner` therefore needs its mask images supplied from +outside the pipeline — point its second `INPUT_DIR` entry at a directory of tile +mask images matching its `NUMBERING_SCHEME`. The healsparse-native replacement +for this whole procedure (an n_epoch / n_pointings survey-window map built from +the maps directly) is issue #797. +``` ```{note} Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see [VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The `random_cat` module itself is current; -the input-staging and joint-mask steps now overlap with -[`sp_validation`](https://github.com/CosmoStat/sp_validation). The steps are -retained for reference. +helper, which is no longer shipped. The input-staging and joint-mask steps now +overlap with [`sp_validation`](https://github.com/CosmoStat/sp_validation). The +steps are retained for reference. ``` ## Set up @@ -41,46 +50,14 @@ If not, we can just download the headers to gain significant download time. shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini ``` -### Check pixel mask files - -Make sure that all pixel mask files are present. If they have been downloaded from ``vos`` as ``.tgz`` files, -type -```bash -canfar_avail_results -i tile_numbers.txt --input_path . -v -m -o missing_mask.txt -``` -In case of missing mask files, check whether they are present in the ``vos`` remote directory, -```bash -canfar_avail_results -i tile_numbers.txt --input_path vos:cfis/vos-path/to/results -v -m -``` -If missing on ``vos``, process those tiles. If processing only up the the mask is necessary, -the following steps can be carried out, -```bash -job_sp -j 7 TILE_ID -job_sp -j 128 TILE_ID -``` -The first command processes the tile up to the mask; the second line uploads the mask files -to ``vos``. +### Stage the pixel mask files -Now, download the missing masks with -```bash -canfar_download_results -i missing_mask.txt --input_vos vos-path/to/results -m -v -``` -Untar .tgz files if required, -```bash -while read p; do tar xvf pipeline_flag_$p.tgz; done -.fits` +with the committed `config_Rc.ini`). How you obtain them is outside ShapePipe; +older runs of the pipeline's own (now removed) mask module wrote them as +`pipeline_flag--.fits`, and those files still work. ## Create random catalogue and helapix mask per tile diff --git a/example/cfis_image_sims/README.md b/example/cfis_image_sims/README.md index b308cd6a8..fa0e00ac4 100644 --- a/example/cfis_image_sims/README.md +++ b/example/cfis_image_sims/README.md @@ -18,7 +18,10 @@ those cases are documented in the last column below and, at more length, under Every row is derived from the two bash scripts. "Module(s)" is the ShapePipe runner(s) the selected `.ini` names; "`.ini` selected" is what `job_sp_canfar_v2.0.bash` picks for that bit under sim settings -(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`, `star_cat_for_mask=onthefly`). +(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`). Bit 32 (mask exposures) is +gone: ShapePipe generates no masks (PR #847), so the bash scripts' +`star_cat_for_mask` setting and the `config_*_Ma_*.ini` configs it selected no +longer exist. | Bit | Stage | Module(s) | `.ini` selected (sim settings) | Sim special-casing | |----:|-------|-----------|--------------------------------|--------------------| @@ -27,10 +30,9 @@ runner(s) the selected `.ini` names; "`.ini` selected" is what | 4 | find exposures | `find_exposures_runner` | `config_tile_Fe.ini` | — | | 8 | retrieve exposure images | `get_images_runner` | `config_exp_Gie_symlink.ini` | symlink retrieval; completeness check expects 3 files vs. 6 for data | | 16 | split exposures, merge WCS headers | `split_exp_runner` | `config_exp_Sp.ini` | — | -| 32 | mask exposures | `mask_runner` | `config_exp_Ma_onthefly.ini` | — | | 64 | exposure PSF model | *(none — placeholder)* | *(none)* | **Placeholder.** For data this runs full exposure PSF modelling. For sims run_job writes a placeholder log and does nothing here; the sim PSF (`fake_psf_runner`) actually runs inside bit 512 | | 128 | merge exposure WCS headers → tile sqlite log | `merge_headers_runner` | `config_tile_Mh_exp.ini` | — | -| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx_nomask.ini` | `tile_det` is forced to `sx`, so the SExtractor-no-mask branch is always taken (the `uc` external-catalogue branch is never reached for sims) | +| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx.ini` | `tile_det` is forced to `sx`, so the SExtractor branch is always taken (the `uc` external-catalogue branch is never reached for sims). Tiles carry no flag image — ShapePipe generates no masks — so this runs with `FLAG_IMAGE = False` | | 512 | fake PSF + postage stamps | `fake_psf_runner`, then `vignetmaker_runner` ×2 | `config_exp_psfex.ini` (fake PSF), then `config_tile_PiViVi_canfar_sx.ini` (vignets) | **Two sub-runs.** run_job first calls the job script with `-j 64` → `config_exp_psfex.ini`, which despite its name runs `fake_psf_runner` (needs the sexcat from bit 256; run dir `run_sp_tile_fpsf`), then `-j 512` → `config_tile_PiViVi_canfar_sx.ini` for the two `vignetmaker_runner` runs. Data instead runs `psfex_interp_runner` + vignets here | | 1024 | multi-epoch shape measurement | `ngmix_runner` | `config_tile_Ng_batch_psfex_sx.ini` | — | | 2048 | create final catalogue | `make_cat_runner` | `config_tile_Mc_psfex.ini` | — | diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini index 28ed565af..26564e4fa 100644 --- a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -130,7 +130,7 @@ ME_IMAGE_PATTERN = flag, image, weight [VIGNETMAKER_RUNNER_RUN_3] # Cut per-object coadd-frame segmentation stamps from the tile SExtractor -# SEGMENTATION check image (config_tile_Sx_nomask.ini: CHECKIMAGE = BACKGROUND, +# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, # SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode # guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN # centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg diff --git a/pyproject.toml b/pyproject.toml index 5388adc86..7ec4c0528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,11 @@ requires-python = ">=3.12" # the code actually requires the newer API. dependencies = [ "astropy>=7.0", # major 6 → 7 - "astroquery", "canfar", "cs_util>=0.2.1", "galsim>=2.8", "h5py", "healsparse", - "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", diff --git a/scripts/README.rst b/scripts/README.rst index 1d3e265a6..2b8067cb0 100644 --- a/scripts/README.rst +++ b/scripts/README.rst @@ -9,7 +9,6 @@ Python scripts ============== 1. `create_log_exp_headers`_ -2. `create_star_cat`_ create_log_exp_headers ====================== @@ -18,11 +17,3 @@ This as to run after the module `split_exp_runner` it will create a master log file containing all the WCS information for each CCDs of each single exposures. To run the script : `python create_log_exp_headers.py path/to/split_exp_runner/output path/to/srcipt/output_dir` - -create_star_cat -=============== - -This script create all the star catalogs required to run the mask module for a -computational node without internet access. -To run the script : -`python create_star_cat.py path/to/image_dir path/to/script/output_dir` diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py index 62fbf0564..cb9afcb0b 100755 --- a/scripts/python/canfar_avail_results.py +++ b/scripts/python/canfar_avail_results.py @@ -113,13 +113,6 @@ def parse_options(p_def): action="store_true", help="only check final catalogues", ) - parser.add_option( - "-m", - "--mask_only", - dest="mask_only", - action="store_true", - help="only check mask files (pipeline_flag)", - ) parser.add_option( "-x", "--extension", @@ -159,10 +152,6 @@ def check_options(options): print("Invalid PSF model '{}'".format(options.psf)) return False - if options.final_only and options.mask_only: - print("One one of the options '-f' or '-m' can be given") - return False - return True @@ -379,13 +368,10 @@ def main(argv=None): if param.final_only: result_base_names = ["final_cat"] - elif param.mask_only: - result_base_names = ["pipeline_flag"] else: result_base_names = [] types = [ "final_cat", - "pipeline_flag", "logs", "setools_mask", "setools_stat", diff --git a/uv.lock b/uv.lock index a4d20ff52..27fe9cd5b 100644 --- a/uv.lock +++ b/uv.lock @@ -168,24 +168,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/27/0775134a2a939ddf7a073e77e66110b671e99cefbc36fb57b57851fd109a/astropy_iers_data-0.2026.8.10.0.32.39-py3-none-any.whl", hash = "sha256:04239afd6e9615165b84da1b84d73f03ca473f0ddf53b516c260e27349a95d03", size = 1998788, upload-time = "2026-08-10T00:33:34.729Z" }, ] -[[package]] -name = "astroquery" -version = "0.4.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "beautifulsoup4" }, - { name = "html5lib" }, - { name = "keyring" }, - { name = "numpy" }, - { name = "pyvo" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/48/273dbde090e071f9d264d084bc49193d126498d2906172b78febd9d62e28/astroquery-0.4.11.tar.gz", hash = "sha256:5537529bddc7fa07e773d5cd9baca593e3f5d93474edd1914f68e89506042b33", size = 12561055, upload-time = "2025-09-20T04:26:36.744Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/ca/944b328f2c60b83896a9223312e21e46cdc1fef57565e853da4544ff6a8e/astroquery-0.4.11-py3-none-any.whl", hash = "sha256:e34f114b285dd07a10ddb2065ebce829b01b0e740fd89dbc81a3077808e24b2d", size = 11139417, upload-time = "2025-09-20T04:26:31.881Z" }, -] - [[package]] name = "asttokens" version = "3.0.2" @@ -1151,19 +1133,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, ] -[[package]] -name = "html5lib" -version = "1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173, upload-time = "2020-06-22T23:32:36.781Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" @@ -2951,19 +2920,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, ] -[[package]] -name = "pyvo" -version = "1.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/35/12c0f4fa0879316837ac56f275942f006f0735e5aedba529b4449dddc36f/pyvo-1.9.1.tar.gz", hash = "sha256:2f26c99af7c32f3c34b919e2d14eaf1a95914176d693fb7769773f3ab0b7999d", size = 2167800, upload-time = "2026-06-11T14:44:01.349Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/09/04ff6e8beaa6cd60a960f925d837c2afa3fffba7860042ab1b52a8358297/pyvo-1.9.1-py3-none-any.whl", hash = "sha256:098648d00943440f56c1d00ac76330433578a0725ea6187afaa6bae08545bb53", size = 1152763, upload-time = "2026-06-11T14:43:59.016Z" }, -] - [[package]] name = "pywavelets" version = "1.9.0" @@ -3442,11 +3398,11 @@ version = "1.1.0" source = { editable = "." } dependencies = [ { name = "astropy" }, - { name = "astroquery" }, { name = "canfar" }, { name = "cs-util" }, { name = "galsim" }, { name = "h5py" }, + { name = "healsparse" }, { name = "joblib" }, { name = "matplotlib" }, { name = "mccd" }, @@ -3518,13 +3474,13 @@ test = [ [package.metadata] requires-dist = [ { name = "astropy", specifier = ">=7.0" }, - { name = "astroquery" }, { name = "build", marker = "extra == 'release'" }, { name = "canfar" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util?branch=develop" }, { name = "fitsio", marker = "extra == 'fitsio'" }, { name = "galsim", specifier = ">=2.8" }, { name = "h5py" }, + { name = "healsparse" }, { name = "hypothesis", marker = "extra == 'test'", specifier = ">=6.155.2" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=9.14.1" }, { name = "joblib", specifier = ">=1.4" }, diff --git a/workflow/README.md b/workflow/README.md index 08bfd2291..ac997afef 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,8 +22,7 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the -# star-catalogue cache root). +# Edit workflow/config.yaml: tile_list, run_dir, container. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -154,8 +153,8 @@ workflow/ bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures - exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) - tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + exposure.smk per-exposure: get_images, split, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) build_index.py prepare-phase run_index.sqlite builder (plain script) @@ -196,16 +195,19 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee committed under `workflow/config/cfis/` and version with the rules that set the env vars they interpolate — there is no `config_src` knob, and no per-unit config symlink; `$SP_CONFIG` points straight at the committed directory. -- **Mask star catalogues are built in the DAG.** `exp_star_cat` runs one Vizier - cone query per exposure into the run-independent cache at `star_cats:`, then - fans it out into a real per-unit `star_cat_exp/` directory of 40 per-CCD - symlinks, which `exp_mask` consumes. The directory must be per-unit and real: - the file handler intersects the image numbers it finds across a config's - `INPUT_DIR`s, so a symlink to the whole cache contributes every other - exposure's numbers and the intersection comes out empty. It is a `localrule`, - so the queries run serially in the head process — CDS is never hammered, and - the scheduler never sees a six-second job. The cache makes reruns and later - campaigns free. +- **There is no masking stage, on either side.** ShapePipe generates no masks + (PR #847). The one mask that reaches pixels is the instrument flag image + delivered with each exposure, which `exp_split` splits per CCD beside image + and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star + halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed + healsparse maps and QUERIED once per object: the `mask_query` module writes a + `FLAG_EXT` column onto each CCD's detection catalogue for setools' star cut + (inside `exp_psf`), and `make_cat` writes one `MASK_` column per band + onto the final catalogue (inside `tile_make_cat`). Map paths are config, not + code, so regenerated products cost a config edit. Nothing is fetched from a + catalogue server, staged, or rasterized, which is why the old + `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are + gone. - **The index is parse-time data, never a rule input.** Appending tiles changes which jobs exist without invalidating completed work. - **Exposure products are not `temp()`.** Exposures overlap tiles, so From 897febc1cffd55b0ab8fe17a686f12d3cff4c5c7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:48:53 -0400 Subject: [PATCH 10/17] feat(mask_query): narrow the shipped PSF-star diet to the star-body map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diet is settled: instrument flags (IMAFLAGS_ISO, already read by SExtractor) plus the UNIONS star-body map (bit 2, mask_ugriz_nside131072_n4.hsp), and nothing else. The committed [MASK_QUERY_RUNNER] examples in config_exp_psfex.ini (both copies) and config_exp_mccd.ini now name exactly that one map instead of the two-map placeholder. Halo bits 0 and 1 stay out on purpose — halos flag objects for the final catalogue, they do not reject PSF stars (mask-force telecon, 2026-07-21) — and MaxiMask is out too. The module docstring says so under its own heading, so that a reader who finds one path in the config knows it is a decision rather than an unfinished list, and knows widening it is a config edit and no code. That asymmetry also explains the contract: MASK_PATHS is a path list rather than a bit mask because the UNIONS products are one boolean map per bit, so choosing bits is choosing files. MASK_BITS survives for integer maps that pack several bits into one file, and its commented example drops from 1028 to 4 to match the diet. The setools header and the tutorial's Masks section carry the same note. Configs re-validated in the container (0 bad, all MODULE lists resolve) and the 10 mask tests still pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_tutorial.md | 4 +++- example/cfis/config_exp_mccd.ini | 20 ++++++++++++++----- example/cfis/config_exp_psfex.ini | 19 ++++++++++++------ example/cfis/star_selection.setools | 7 +++++-- .../modules/mask_query_package/__init__.py | 15 ++++++++++++++ workflow/config/cfis/config_exp_psfex.ini | 19 ++++++++++++------ 6 files changed, 64 insertions(+), 20 deletions(-) diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 36dde0ccd..a4db96579 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -197,7 +197,9 @@ pixels. Two modules do the querying, from the same shared lookup (`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and `setools` on the single-exposure single-CCD catalogues and writes one integer `FLAG_EXT` column (0 = clean), which `star_selection.setools` cuts on so that -masked objects never enter the PSF star sample; `make_cat` writes one +masked objects never enter the PSF star sample — a deliberately narrow diet, +the star-body map (bit 2) only, since halos flag objects without disqualifying +them as PSF stars; `make_cat` writes one `MASK_` column per band onto the final tile catalogue, carrying the map value verbatim so downstream selections choose their own cuts. Map paths and bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 8e9ffbd8f..68dcfbeef 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -137,11 +137,21 @@ FILE_PATTERN = sexcat_sexcat NUMBERING_SCHEME = -0000000-0 -# External healsparse masks queried at each detection's (RA, Dec); see the -# mask_query module docstring. star_selection.setools cuts on FLAG_EXT == 0. -MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp - -; MASK_BITS = 1028 +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets FLAG_EXT, which star_selection.setools cuts on as +# FLAG_EXT == 0. Comma-separated. +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 [SETOOLS_RUNNER] diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 10566d5ed..2f1c8e353 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -139,14 +139,21 @@ FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 -# External healsparse masks queried at each detection's (RA, Dec). Any map -# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which -# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. -MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets FLAG_EXT, which star_selection.setools cuts on as +# FLAG_EXT == 0. Comma-separated. +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, -# any nonzero value flags. Boolean maps ignore it. -; MASK_BITS = 1028 +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 [SETOOLS_RUNNER] diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index f35043d99..230017053 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -4,8 +4,11 @@ ## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), ## delivered with the exposure and read by SExtractor; ## FLAG_EXT == 0 the external healsparse masks, queried per detection by -## the mask_query module (which map bits reach FLAG_EXT is -## that module's MASK_PATHS / MASK_BITS config). +## the mask_query module. Which maps reach FLAG_EXT is that +## module's MASK_PATHS config, and the shipped diet is +## deliberately narrow: the star-body map (bit 2) only, no +## halos (they flag, they do not reject stars) and no +## MaxiMask. ## SETools expressions have no bitwise operators, so mask_query does the bit ## selection and this file only tests for zero. diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index b4b1d92c1..087e50722 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -34,6 +34,21 @@ ``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is written once. That module's docstring documents the off-coverage convention. +The diet is deliberately narrow +=============================== + +``MASK_PATHS`` is a *list of maps to reject PSF stars on*, not a list of every +mask that exists. The committed configs name exactly one map — the UNIONS +star-body product (bit 2) — beside the instrument flags SExtractor already +reads. Halo bits 0 and 1 are excluded on purpose: halos flag objects for the +final catalogue, they do not reject PSF stars (mask-force telecon, 2026-07-21). +MaxiMask is not in the diet either. + +Widening it costs a config edit and no code — add a path. That is why the +contract is a path list rather than a bit mask: the UNIONS products are one +boolean map per bit, so choosing bits *is* choosing files, and ``MASK_BITS`` +exists only for integer maps that pack several bits into one file. + Module-specific config file entries =================================== diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 08f45f99b..ffcac86b6 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -138,14 +138,21 @@ FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 -# External healsparse masks queried at each detection's (RA, Dec). Any map -# that is True (boolean) or nonzero (integer) there sets FLAG_EXT, which -# star_selection.setools cuts on with FLAG_EXT == 0. Comma-separated paths. -MASK_PATHS = $SP_CONFIG/mask_star.hsp, $SP_CONFIG/mask_maximask.hsp +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets FLAG_EXT, which star_selection.setools cuts on as +# FLAG_EXT == 0. Comma-separated. +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, -# any nonzero value flags. Boolean maps ignore it. -; MASK_BITS = 1028 +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 [SETOOLS_RUNNER] From ba2271a55cd57f425962679b8830865c27b91636 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 10:53:36 -0400 Subject: [PATCH 11/17] =?UTF-8?q?fix(mask):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20CI=20smoke,=20SEGMENTATION=20checkimage,=20canfar=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups from review. CI. deploy-image.yml's runtime binary smoke still ran `weightwatcher --version` after the Dockerfile dropped the package, so the very next image build would have failed on a tool nothing calls. Step line and the comment's mention both go. SEGMENTATION, and this one was a real regression I introduced. Folding config_tile_Sx_nomask.ini into config_tile_Sx.ini carried the nomask variant's `CHECKIMAGE = BACKGROUND` over the masked config's `BACKGROUND, SEGMENTATION`. The segmentation check image is not masking furniture: vignetmaker cuts per-object segmentation stamps from it (config_tile_PiViVi_canfar_sx.ini FILE_PATTERN = sexcat, segmentation), and ngmix's uberseg blend handling raises at construction without them (ngmix.py ~496, shapepipe#776). Restored in both copies. The sims config already had it, which is why nothing else caught this. Docs. pipeline_canfar.md's "Collate star catalogues" section drove collate_star_cat.py, deleted with the star-catalogue tooling. The recipe is replaced with a warning that names what is missing rather than a silent cut: the merge step below it consumes validation_psf_conv files that now have no producer in this repo, and the star_cat/ + `combine_runs.bash -c psf_conv` staging around it does not apply either. The PSF measurement itself is unchanged — shapes are measured in sky coordinates during interpolation — so the collation is a pass-through that is straightforward to rebuild. No other docs page referenced a deleted script. Re-verified in the container: 30/30 module runners import, 65 configs' MODULE lists resolve (0 bad), the 10 mask tests pass, the workflow still parses and builds its DAG, and deploy-image.yml is valid YAML. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- .github/workflows/deploy-image.yml | 3 +- docs/source/pipeline_canfar.md | 72 ++++++------------------- example/cfis/config_tile_Sx.ini | 2 +- workflow/config/cfis/config_tile_Sx.ini | 2 +- 4 files changed, 20 insertions(+), 59 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 53e67dcca..6de369a66 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -83,12 +83,11 @@ jobs: # Smoke-test the binaries baked into the runtime image. Catches the # class of regression where the image builds but a runtime tool - # (sextractor, weightwatcher) is missing or unrunnable. + # (sextractor, psfex) is missing or unrunnable. - name: Test runtime — binaries run: | IMAGE=$(echo "${{ steps.meta-runtime.outputs.tags }}" | head -n1) docker run --rm "$IMAGE" source-extractor --version - docker run --rm "$IMAGE" weightwatcher --version docker run --rm "$IMAGE" psfex --version - name: Test runtime — shapepipe entry point (read-only fs) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 926d36578..108f948f4 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -309,61 +309,23 @@ shapepipe_run -c $SP_CONFIG/config_Pl_$psf.ini #### Collate star catalogues -Collate all input validation PSF files into star catalogues, gathering positions -(X/Y/RA/DEC) and the MCCD CCD id. - -Note: HSM shapes are no longer rotated into world coordinates at this step. The -PSF/star ellipticities and sizes are now measured directly in sky coordinates -during PSF interpolation (galsim `FindAdaptiveMom(use_sky_coords=True)`), so -`collate_star_cat.py` only collates and passes the shapes through. - -> **v2.0 is patch-less.** Runs up to `v1.6` are organised in sky patches -> `P1`..`P`, each patch a run directory. `v2.0` removes the patch concept: a -> single run root, outputs directly under it. `v2.0` is the default; select an -> older layout with `-V` (e.g. `-V v1.6`). For `v2.0` the patch loop and the -> `-P` option no longer apply, and the patch token drops from the output -> filename (`validation_psf_conv-.fits` instead of -> `validation_psf_conv--.fits`). - -```bash -cd /path/to/version -mkdir star_cat -cd star_cat -``` - -For `v2.0` (the default), run once against the patch-less run root, producing -files `validation_psf_conv-.fits`: - -```bash -collate_star_cat.py -i .. -v -``` - -For `v1.x`, pass the version explicitly and run once per patch, creating a -directory per patch `P?` and producing files -`validation_psf_conv--.fits` (for the v1.4 setup only one file): - -```bash -collate_star_cat.py -i .. -V v1.6 -P $patchnum -v -``` - -Combine previously created files as links within one ShapePipe run directory (for the v1.4 setup only one link). -First (and optiohnal), create a subdir for a run and link to the input patches: - -```bash -cd /path/to/version/star_cat -mkdir v1.6 -ln -s ../P1 -ln -s ../P2 -... -``` - -Next, create links to all `validation_conv` runs: - -```bash -combine_runs.bash -p psfex -c psf_conv -``` - -Merge all converted star catalogues and create `final-starcat.fits`: +```{warning} +**This step no longer ships.** `collate_star_cat.py` gathered each validation +PSF run's star positions (X/Y/RA/DEC) and MCCD CCD id into the +`validation_psf_conv-*.fits` files the merge below consumes, and it was deleted +with the star-catalogue tooling the removed mask module needed (PR #847). The +merge step that follows therefore has no producer for its input in this repo +until the collation is reimplemented, and the `star_cat/` + `combine_runs.bash +-c psf_conv` staging around it does not apply either. + +Nothing about the PSF measurement itself changed: HSM shapes are measured +directly in sky coordinates during PSF interpolation (galsim +`FindAdaptiveMom(use_sky_coords=True)`), which is why the collation was a +pass-through for shapes and is straightforward to rebuild. +``` + +Merge all converted star catalogues and create `final-starcat.fits` (this +reads the `validation_psf_conv` files the collation above used to produce): ```bash export SP_RUN=`pwd` diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini index 9859e4237..80350eead 100644 --- a/example/cfis/config_tile_Sx.ini +++ b/example/cfis/config_tile_Sx.ini @@ -102,7 +102,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 0ce9ea226..5487f547c 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -101,7 +101,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat From f7d1fd663d19c96fab76b6a0e7091c5f11b28392 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:00:37 -0400 Subject: [PATCH 12/17] perf(mask_query): read only the coverage pixels a catalogue touches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five review items, all in the query path. PARTIAL READS. query_map did HealSparseMap.read(path) on a 583 MB map, per CCD — ~22 GB of I/O per 40-CCD exposure to answer questions about 0.06 deg². It now reads HealSparseCoverage first, computes the coverage pixels the positions touch with hpgeom.angle_to_pixel at the map's nside_coverage, and loads only those. make_cat gets this for free, sharing the primitive. Measured on the DR6 star map for 2000 positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, IDENTICAL values. 125x faster, 21x less memory. The full-read footprint is 3.3 GiB — under the 4 GB mark — and the partial read adds ~0.15 GB to a rule already asking for 16 GB, so exp_psf's mem_mb is left alone. Per exposure this is ~4 s of map reading instead of ~8.5 min. test_partial_read_matches_full pins partial == full for both map dtypes. Two things the reviewer's sketch did not anticipate, both found by probing the real map rather than reasoning. (a) healsparse RAISES when no requested pixel is in the coverage map, so the all-off-coverage case is answered without asking it: one small probe read for dtype and sentinel, then the sentinel everywhere. (b) The neighbours padding is insurance, not necessity — every position falls in exactly one coverage pixel and all are requested — but at nside_coverage=128 it costs under a megabyte, so it stays. BOOLEAN OFF-COVERAGE. n_off was hardcoded to 0 for boolean maps, so a map that misses an exposure logged "0 flagged, 0 outside coverage" — indistinguishable from clean. valid_mask=True cannot fix this: the UNIONS products are BOOLEAN, and healsparse stores only the True pixels, so valid_mask returns the value itself (verified against mask_r_nside131072_n4.hsp). Coverage now comes from the coverage mask, which works for both dtypes, and all-off-coverage logs a warning saying the zero column means "the map does not reach here", not "clean". Also: FLAG_EXT's docstring no longer claims the value says which bits fired — true only for integer maps, since boolean maps can only contribute 1. config_Rc.ini's mask INPUT_DIR placeholder becomes rather than a $SP_CONFIG path that does not exist. A zero-detection CCD no longer raises on data["XWIN_WORLD"] — it writes an empty FLAG_EXT column, matching the sparse-CCD tolerance setools and the completeness floor already carry. hpgeom returns to pyproject.toml, now imported directly rather than via healsparse. pipeline_canfar.md's HSM sky-coordinates paragraph is lifted out of the collate_star_cat deletion warning into standing prose, pointing at the test that pins the convention. Verified: 15/15 mask tests pass, 30/30 runners import, 65 configs resolve, the workflow still builds its DAG, uv.lock regenerated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 10 +- example/cfis/config_Rc.ini | 2 +- pyproject.toml | 1 + .../modules/mask_query_package/__init__.py | 10 +- .../modules/mask_query_package/mask_query.py | 20 ++- src/shapepipe/utilities/mask_query.py | 163 ++++++++++++++++-- tests/module/test_mask_query.py | 113 +++++++++++- uv.lock | 2 + 8 files changed, 288 insertions(+), 33 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 108f948f4..350c4a403 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -317,12 +317,14 @@ with the star-catalogue tooling the removed mask module needed (PR #847). The merge step that follows therefore has no producer for its input in this repo until the collation is reimplemented, and the `star_cat/` + `combine_runs.bash -c psf_conv` staging around it does not apply either. +``` -Nothing about the PSF measurement itself changed: HSM shapes are measured +HSM shapes are **not** rotated into world coordinates at this step, and were not +before it was removed. The PSF and star ellipticities and sizes are measured directly in sky coordinates during PSF interpolation (galsim -`FindAdaptiveMom(use_sky_coords=True)`), which is why the collation was a -pass-through for shapes and is straightforward to rebuild. -``` +`FindAdaptiveMom(use_sky_coords=True)`) — see `tests/module/test_hsm_sky_coords.py`, +which pins that convention. The collation was a pass-through for shapes, which is +both why it carried no science and why it is straightforward to rebuild. Merge all converted star catalogues and create `final-starcat.fits` (this reads the `validation_psf_conv` files the collation above used to produce): diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini index 5d04e8e37..9f55f1812 100644 --- a/example/cfis/config_Rc.ini +++ b/example/cfis/config_Rc.ini @@ -57,7 +57,7 @@ TIMEOUT = 96:00:00 # masks. Point the second entry at a directory of tile mask images matching # NUMBERING_SCHEME below (the healsparse-native replacement for this module is # the survey-window work, issue #797). -INPUT_DIR = last:get_images_runner, $SP_CONFIG/tile_masks +INPUT_DIR = last:get_images_runner, FILE_PATTERN = CFIS_image, mask diff --git a/pyproject.toml b/pyproject.toml index 7ec4c0528..4c5fd7e02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "galsim>=2.8", "h5py", "healsparse", + "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index 087e50722..0391e5393 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -20,9 +20,13 @@ maps, writing one integer column: ``FLAG_EXT`` - ``0`` for an object no configured map flags, nonzero otherwise. The nonzero - value is the bitwise OR of the contributing map values, so it says *which* - bits fired, but nothing downstream is required to read it that way. + ``0`` for an object no configured map flags, nonzero otherwise. What the + nonzero value *is* depends on the maps: a boolean map — which is what the + UNIONS per-bit products are, and what the shipped config names — can only + contribute ``1``, so with those the column is 0/1 and says nothing about + which map fired. An integer map contributes its own value (optionally + ``& MASK_BITS``), and contributions are OR-ed, so a bit-packed map does + carry its bits through. Nothing downstream reads more than ``== 0``. The single column exists because ``setools`` expressions support only ``< > <= >= == !=`` — no bitwise operators — so the bit selection has to diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py index 16f3c5ff5..08e153a00 100644 --- a/src/shapepipe/modules/mask_query_package/mask_query.py +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -67,8 +67,24 @@ def process(self): ) ori_cat.open() data = ori_cat.get_data() - ra = np.copy(data["XWIN_WORLD"]) - dec = np.copy(data["YWIN_WORLD"]) + + # A CCD SExtractor found nothing on is tolerated all along this chain + # (setools' ~0.2% attrition, psfex_interp's floor=0 warn), so it must + # not be an error here either. An empty LDAC table has no columns to + # index, so read the positions only when there are rows, and still + # publish an output file — a missing sexcat_ext would look to the file + # handler like a crash rather than like an empty CCD. + if len(data) == 0: + ra = np.zeros(0) + dec = np.zeros(0) + if self._w_log is not None: + self._w_log.info( + "No detections in " + + f"{self._sexcat_path}; writing an empty FLAG_EXT column" + ) + else: + ra = np.copy(data["XWIN_WORLD"]) + dec = np.copy(data["YWIN_WORLD"]) flag = mask_query_util.flag_positions( self._mask_paths, diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index 0cc64255d..c3b150006 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -17,6 +17,28 @@ flagged (nonzero)" so that ``setools`` — whose expression language has no bitwise operators — can cut on ``FLAG_EXT == 0``. +Partial reads +------------- +Never read a whole map. The UNIONS products are ~550 MB each and ``mask_query`` +runs PER CCD, so a full read would cost ~22 GB of I/O per 40-CCD exposure to +answer questions about a 0.06 deg² footprint. Instead the coverage index is +read first (``HealSparseCoverage``, a few hundred kB), the coverage pixels the +catalogue actually touches are computed with ``hpgeom.angle_to_pixel`` at the +map's ``nside_coverage``, and only those get loaded. + +Every queried position falls inside exactly one coverage pixel and all of them +are requested, so the padding by ``hpgeom.neighbors`` is insurance rather +than necessity — at ``nside_coverage=128`` a coverage pixel of the star map +is ~23 kB, so the padding costs under a megabyte and buys immunity to any +edge convention we did not think of. ``test_partial_read_matches_full`` is +what actually holds the two paths equal. + +Measured on the DR6 star map (``mask_r_nside131072_n4.hsp``, 583 MB, +``nside_coverage=128``) for 2000 positions in one CCD-sized box, one process +each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, identical +values. Per 40-CCD exposure that is ~4 s against ~8.5 min of map reading, and +the query adds ~0.15 GB to a rule already asking for 16 GB. + Coverage -------- ``healsparse.HealSparseMap.get_values_pos`` returns a map's *sentinel* for @@ -24,11 +46,20 @@ ``-1`` for integer maps. ``make_cat`` passes that sentinel through verbatim, which is the documented off-map flag for the final catalogue. -``flag_positions`` instead treats off-coverage as **not flagged**, for both map -kinds. This makes the integer case agree with the boolean case (whose sentinel -is literally ``False``) rather than diverge from it, and it keeps a map whose +Coverage is reported from the COVERAGE MASK, not ``valid_mask=True``. For a +boolean map — which is what the UNIONS per-bit products are — healsparse +stores only the ``True`` pixels, so ``valid_mask`` returns the value itself +and cannot tell "inside the footprint and clean" from "outside it entirely". +The coverage mask can, at ``nside_coverage`` resolution, which is the scale the +question is asked at anyway: does this map reach this exposure at all? + +``flag_positions`` treats off-coverage as **not flagged**, for both map kinds. +This makes the integer case agree with the boolean case (whose sentinel is +literally ``False``) rather than diverge from it, and it keeps a map whose coverage does not reach an exposure from silently rejecting every star on it. -Off-coverage counts are logged so the situation is visible rather than silent. +That is also why the all-off-coverage case is logged as a WARNING: it is +indistinguishable from "nothing is masked here" in the output column, so it has +to be distinguishable in the log. :Author: Claude Fable 5, for PR #847 @@ -56,10 +87,106 @@ def parse_map_paths(paths_str): return [path.strip() for path in paths_str.split(",") if path.strip()] +def _covering_pixels(coverage, ra, dec): + """Covering Pixels. + + The map's coverage pixels touched by these positions, padded by their + neighbours and intersected with what the map actually holds. + + Parameters + ---------- + coverage : healsparse.HealSparseCoverage + Coverage index of the map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Coverage pixel indices to load, possibly empty + + """ + import hpgeom + + nside_coverage = coverage.nside_coverage + touched = np.unique( + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ) + padded = np.unique( + np.concatenate( + [touched, hpgeom.neighbors(nside_coverage, touched).ravel()] + ) + ) + # neighbors() returns -1 for a non-existent neighbour. + padded = padded[padded >= 0] + + return padded[coverage.coverage_mask[padded]] + + +def query_map_coverage(path, ra, dec): + """Query Map With Coverage. + + Read only the part of a healsparse map these positions need, and return + both its value at each position and whether each position is inside the + map's coverage. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + tuple + ``(values, in_coverage)`` — the map value at each position (the map's + sentinel outside coverage) and a boolean array, both of length + ``len(ra)`` + + """ + import healsparse + import hpgeom + + ra = np.asarray(ra) + dec = np.asarray(dec) + + coverage = healsparse.HealSparseCoverage.read(path) + nside_coverage = coverage.nside_coverage + + in_coverage = coverage.coverage_mask[ + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ] + + pixels = _covering_pixels(coverage, ra, dec) + + if pixels.size == 0: + # healsparse raises when no requested pixel is in the coverage map, so + # the empty case is answered without asking it: load one arbitrary + # coverage pixel purely to learn the dtype and sentinel, and return + # that sentinel everywhere. Same answer, one small read. + probe = int(np.flatnonzero(coverage.coverage_mask)[0]) + mask_map = healsparse.HealSparseMap.read(path, pixels=[probe]) + values = np.full(ra.size, mask_map._sentinel, dtype=mask_map.dtype) + return values, in_coverage + + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(pixel) for pixel in pixels] + ) + values = np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + + return values, in_coverage + + def query_map(path, ra, dec): """Query Map. - Read a healsparse map and return its value at each world position. + Return a healsparse map's value at each world position, reading only the + coverage pixels those positions touch. Parameters ---------- @@ -77,11 +204,9 @@ def query_map(path, ra, dec): the map's sentinel value """ - import healsparse - - mask_map = healsparse.HealSparseMap.read(path) + values, _ = query_map_coverage(path, ra, dec) - return np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + return values def flag_positions(paths, ra, dec, bits=None, w_log=None): @@ -123,30 +248,40 @@ def flag_positions(paths, ra, dec, bits=None, w_log=None): dec = np.asarray(dec) flag = np.zeros(ra.size, dtype=np.int64) + if ra.size == 0: + return flag + for path in paths: - values = query_map(path, ra, dec) + values, in_coverage = query_map_coverage(path, ra, dec) if values.dtype == bool: contribution = values.astype(np.int64) - n_off = 0 else: integer = values.astype(np.int64) # Off-coverage: the sentinel, negative by healsparse convention. # Zeroed rather than OR-ed in, see this module's docstring. - off_coverage = integer < 0 - n_off = int(np.count_nonzero(off_coverage)) - integer = np.where(off_coverage, 0, integer) + integer = np.where(integer < 0, 0, integer) if bits is not None: integer &= bits contribution = integer flag |= contribution + n_off = int(np.count_nonzero(~in_coverage)) if w_log is not None: w_log.info( f"Mask query {path}: " f"{int(np.count_nonzero(contribution))}/{ra.size} objects " f"flagged, {n_off} outside coverage" ) + if n_off == ra.size: + # Every object reads the sentinel, so the column is all-zero + # and looks exactly like "nothing is masked here". Say it. + w_log.warning( + f"Mask query {path}: NO object is inside this map's" + + " coverage — the resulting FLAG_EXT contribution is" + + " zero everywhere because the map does not reach these" + + " positions, not because they are clean." + ) return flag diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py index 053dffd1e..779a6bd69 100644 --- a/tests/module/test_mask_query.py +++ b/tests/module/test_mask_query.py @@ -35,8 +35,17 @@ class _NullLogger: - def info(self, *_args, **_kwargs): - pass + """Captures what the module logs, so coverage warnings can be asserted.""" + + def __init__(self): + self.info_msgs = [] + self.warning_msgs = [] + + def info(self, msg, *_a, **_kw): + self.info_msgs.append(str(msg)) + + def warning(self, msg, *_a, **_kw): + self.warning_msgs.append(str(msg)) def _write_map(path, value, dtype=np.int16, n_covered=2): @@ -73,7 +82,7 @@ def _write_bool_map(path, n_covered=2): return str(path) -def _write_sexcat(path): +def _write_sexcat(path, n=None): """Write a synthetic LDAC SExtractor catalogue of known positions. Written with astropy rather than ``FITSCatalogue.save_as_fits``, because @@ -81,6 +90,7 @@ def _write_sexcat(path): to copy the ``LDAC_IMHEAD`` HDU from. The three-HDU layout below is what SExtractor writes and what ``SEx_catalogue=True`` (``hdu_no=2``) indexes. """ + n = len(RA) if n is None else n imhead = fits.BinTableHDU.from_columns( [ fits.Column( @@ -91,13 +101,11 @@ def _write_sexcat(path): ) objects = fits.BinTableHDU.from_columns( [ - fits.Column(name="NUMBER", format="J", array=np.arange(len(RA))), - fits.Column(name="XWIN_WORLD", format="D", array=RA), - fits.Column(name="YWIN_WORLD", format="D", array=DEC), + fits.Column(name="NUMBER", format="J", array=np.arange(n)), + fits.Column(name="XWIN_WORLD", format="D", array=RA[:n]), + fits.Column(name="YWIN_WORLD", format="D", array=DEC[:n]), fits.Column( - name="IMAFLAGS_ISO", - format="J", - array=np.zeros(len(RA), dtype="i4"), + name="IMAFLAGS_ISO", format="J", array=np.zeros(n, dtype="i4") ), ], name="LDAC_OBJECTS", @@ -204,3 +212,90 @@ def test_mask_query_bits_and_all_clean(tmp_path): assert n_flagged == 0 flag, _ = _read(out_path) npt.assert_array_equal(flag, [0, 0, 0, 0]) + + +def test_partial_read_matches_full(tmp_path): + """The partial read returns exactly what a full read of the map returns. + + This is the guarantee that makes the optimisation safe: query_map loads + only the coverage pixels the positions touch, and must be indistinguishable + from HealSparseMap.read(path) at every queried position. + """ + for dtype, writer in ((np.int16, _write_map), (np.bool_, None)): + path = ( + _write_map(tmp_path / f"full_{dtype.__name__}.hsp", 7, n_covered=3) + if writer is not None + else _write_bool_map(tmp_path / "full_bool.hsp", n_covered=3) + ) + full = healsparse.HealSparseMap.read(path) + expected = np.asarray(full.get_values_pos(RA, DEC, lonlat=True)) + npt.assert_array_equal( + mask_query_util.query_map(path, RA, DEC), expected + ) + + +def test_coverage_reported_for_both_map_kinds(tmp_path): + """in_coverage is the coverage mask, not valid_mask. + + For a boolean map healsparse stores only the True pixels, so valid_mask + would equal the value and could not distinguish an unmasked object from one + the map does not reach. The distant object must read as off-coverage while + the near, unflagged ones read as covered. + """ + for path in ( + _write_map(tmp_path / "int.hsp", 4, n_covered=2), + _write_bool_map(tmp_path / "bool.hsp", n_covered=2), + ): + _, in_coverage = mask_query_util.query_map_coverage(path, RA, DEC) + # The 4th position is 190 deg away; the first two are on the map. + assert in_coverage[0] and in_coverage[1] + assert not in_coverage[3] + + +def test_all_off_coverage_warns(tmp_path): + """A map that reaches nothing warns, instead of logging a silent zero.""" + path = _write_map(tmp_path / "elsewhere.hsp", 4, n_covered=2) + far_ra = np.array([200.0, 201.0]) + far_dec = np.array([-40.0, -41.0]) + log = _NullLogger() + + flag = mask_query_util.flag_positions([path], far_ra, far_dec, w_log=log) + + npt.assert_array_equal(flag, [0, 0]) + assert any("NO object is inside" in m for m in log.warning_msgs) + assert any("2 outside coverage" in m for m in log.info_msgs) + # A map that DOES reach the objects must not warn. + log2 = _NullLogger() + mask_query_util.flag_positions([path], RA, DEC, w_log=log2) + assert not log2.warning_msgs + + +def test_flag_positions_empty_input(tmp_path): + """Zero positions is not an error and reads no map.""" + flag = mask_query_util.flag_positions( + [str(tmp_path / "does-not-exist.hsp")], np.zeros(0), np.zeros(0) + ) + assert flag.shape == (0,) + + +def test_mask_query_empty_ccd(tmp_path): + """A CCD with no detections still publishes a catalogue, not an error. + + setools tolerates sparse-CCD attrition and psfex_interp's completeness + floor is 0/warn, so an empty sexcat must flow through rather than raise. + """ + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-2.fits", n=0) + out_path = tmp_path / "sexcat_ext-000-2.fits" + log = _NullLogger() + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=log + ).process() + + assert n_flagged == 0 + assert out_path.exists() + flag, names = _read(out_path) + assert flag.shape == (0,) + assert "FLAG_EXT" in names + assert any("No detections" in m for m in log.info_msgs) diff --git a/uv.lock b/uv.lock index 27fe9cd5b..6a119eea6 100644 --- a/uv.lock +++ b/uv.lock @@ -3403,6 +3403,7 @@ dependencies = [ { name = "galsim" }, { name = "h5py" }, { name = "healsparse" }, + { name = "hpgeom" }, { name = "joblib" }, { name = "matplotlib" }, { name = "mccd" }, @@ -3481,6 +3482,7 @@ requires-dist = [ { name = "galsim", specifier = ">=2.8" }, { name = "h5py" }, { name = "healsparse" }, + { name = "hpgeom" }, { name = "hypothesis", marker = "extra == 'test'", specifier = ">=6.155.2" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=9.14.1" }, { name = "joblib", specifier = ">=1.4" }, From 2cc82eef82485ae27e5ce78c4abaf56d98359cf8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:03:28 -0400 Subject: [PATCH 13/17] =?UTF-8?q?revert(scripts):=20restore=20collate=5Fst?= =?UTF-8?q?ar=5Fcat=20=E2=80=94=20it=20is=20PSF=20validation,=20not=20mask?= =?UTF-8?q?ing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scope error in eb93838a, mine to own: collate_star_cat.py was swept up with the GSC masking star catalogues on a name match. It is the PSF VALIDATION collation — per-exposure validation_psf files gathered into the validation_psf_conv catalogues config_Ms_psfex_conv.ini merges — and has nothing to do with mask generation. Restored byte-identical from origin/feat/snakemake-orchestration, with tests/module/test_collate_star_cat.py, and pipeline_canfar.md's "Collate star catalogues" section put back verbatim in place of the deletion warning ba2271a5 wrote (the HSM sky-coordinates paragraph returns with it, as the section's own prose, which is where it started). Two things asked for in the revert turn out not to be needed, checked rather than assumed: * focal_plane.py and utilities/file_io.py stay deleted. collate_star_cat.py imports nothing from shapepipe at all — only stdlib plus tqdm, joblib, numpy, astropy, galsim and cs_util. focal_plane_disc and write_atomic had exactly two callers between them, create_star_cat.py and star_cats.py, and both are masking and both stay deleted. * There is no scripts/README.rst entry to restore. That file listed create_star_cat (the masking one) and never listed collate_star_cat, so the edit in 1bceb2da was already correct. create_star_cat.py, star_cats.py and utilities/vizier.py remain deleted, as intended. Verified in the container: the 9 restored tests pass alongside the 15 mask ones (24 total), 30/30 runners import, 65 configs resolve, the workflow builds its DAG, and both restored files diff clean against origin. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 74 ++- scripts/python/collate_star_cat.py | 895 ++++++++++++++++++++++++++ tests/module/test_collate_star_cat.py | 70 ++ 3 files changed, 1020 insertions(+), 19 deletions(-) create mode 100755 scripts/python/collate_star_cat.py create mode 100644 tests/module/test_collate_star_cat.py diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 350c4a403..926d36578 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -309,25 +309,61 @@ shapepipe_run -c $SP_CONFIG/config_Pl_$psf.ini #### Collate star catalogues -```{warning} -**This step no longer ships.** `collate_star_cat.py` gathered each validation -PSF run's star positions (X/Y/RA/DEC) and MCCD CCD id into the -`validation_psf_conv-*.fits` files the merge below consumes, and it was deleted -with the star-catalogue tooling the removed mask module needed (PR #847). The -merge step that follows therefore has no producer for its input in this repo -until the collation is reimplemented, and the `star_cat/` + `combine_runs.bash --c psf_conv` staging around it does not apply either. -``` - -HSM shapes are **not** rotated into world coordinates at this step, and were not -before it was removed. The PSF and star ellipticities and sizes are measured -directly in sky coordinates during PSF interpolation (galsim -`FindAdaptiveMom(use_sky_coords=True)`) — see `tests/module/test_hsm_sky_coords.py`, -which pins that convention. The collation was a pass-through for shapes, which is -both why it carried no science and why it is straightforward to rebuild. - -Merge all converted star catalogues and create `final-starcat.fits` (this -reads the `validation_psf_conv` files the collation above used to produce): +Collate all input validation PSF files into star catalogues, gathering positions +(X/Y/RA/DEC) and the MCCD CCD id. + +Note: HSM shapes are no longer rotated into world coordinates at this step. The +PSF/star ellipticities and sizes are now measured directly in sky coordinates +during PSF interpolation (galsim `FindAdaptiveMom(use_sky_coords=True)`), so +`collate_star_cat.py` only collates and passes the shapes through. + +> **v2.0 is patch-less.** Runs up to `v1.6` are organised in sky patches +> `P1`..`P`, each patch a run directory. `v2.0` removes the patch concept: a +> single run root, outputs directly under it. `v2.0` is the default; select an +> older layout with `-V` (e.g. `-V v1.6`). For `v2.0` the patch loop and the +> `-P` option no longer apply, and the patch token drops from the output +> filename (`validation_psf_conv-.fits` instead of +> `validation_psf_conv--.fits`). + +```bash +cd /path/to/version +mkdir star_cat +cd star_cat +``` + +For `v2.0` (the default), run once against the patch-less run root, producing +files `validation_psf_conv-.fits`: + +```bash +collate_star_cat.py -i .. -v +``` + +For `v1.x`, pass the version explicitly and run once per patch, creating a +directory per patch `P?` and producing files +`validation_psf_conv--.fits` (for the v1.4 setup only one file): + +```bash +collate_star_cat.py -i .. -V v1.6 -P $patchnum -v +``` + +Combine previously created files as links within one ShapePipe run directory (for the v1.4 setup only one link). +First (and optiohnal), create a subdir for a run and link to the input patches: + +```bash +cd /path/to/version/star_cat +mkdir v1.6 +ln -s ../P1 +ln -s ../P2 +... +``` + +Next, create links to all `validation_conv` runs: + +```bash +combine_runs.bash -p psfex -c psf_conv +``` + +Merge all converted star catalogues and create `final-starcat.fits`: ```bash export SP_RUN=`pwd` diff --git a/scripts/python/collate_star_cat.py b/scripts/python/collate_star_cat.py new file mode 100755 index 000000000..f7ab25b41 --- /dev/null +++ b/scripts/python/collate_star_cat.py @@ -0,0 +1,895 @@ +#! /usr/bin/env python3 + +"""COLLATE STAR CATALOGUES. + +Collate the per-exposure PSF validation catalogues into star catalogues: gather +positions (X/Y/RA/DEC), assign the MCCD focal-plane CCD id, and merge into +``validation_psf_conv`` FITS files. + +Catalogue layout depends on the major version (``-V``). Runs up to ``v1.6`` are +organised in sky patches (``P1``..``P``): input runs live under +``/P/output`` and outputs are named +``validation_psf_conv--.fits``. ``v2.0`` removes the patch concept: +input runs live under a single ``/output`` root and outputs drop the patch +token (``validation_psf_conv-.fits``). + +HSM ellipticities and sizes are no longer rotated here. PSFEx and the in-repo +MCCD interpolation now measure adaptive moments directly in world coordinates +(galsim ``FindAdaptiveMom(use_sky_coords=True)``), so the WCS-Jacobian shape +rotation this script used to perform is redundant and has been removed. + +Caveat: the MCCD ``PSF_MOM_LIST``/``STAR_MOM_LIST`` columns are produced by the +external ``mccd`` fit-validation code (``mccd.auxiliary_fun.mccd_validation``), +which still measures HSM moments in the pixel frame. Those shapes are therefore +still rotated into world coordinates here, via the WCS-Jacobian rotation, until +``mccd`` itself adopts ``use_sky_coords``. This is the one branch that keeps the +rotation; the in-repo PSFEx and MCCD-interpolation paths measure adaptive +moments directly in world coordinates upstream and pass them straight through. +""" + +import sys +import os +import re +import glob +from tqdm import tqdm +from joblib import Parallel, delayed +import gc + +import numpy as np +from astropy.io import fits +import galsim + +from cs_util import args as cs_args +from cs_util import logging + + +def collate_paths(input_base_dir, output_base_dir, patch): + """Collate Paths. + + Return the ``(input run dir, output dir)`` for a patch. ``patch`` is None + for the patch-less v2.0 layout, which drops the ``P`` token; v1.x + passes the patch number. + + Parameters + ---------- + input_base_dir : str + input base directory + output_base_dir : str + output base directory + patch : str or None + patch number, or None for the patch-less v2.0 layout + + Returns + ------- + tuple + input run directory and output directory + + """ + if patch is None: + return f"{input_base_dir}/output/", output_base_dir + return f"{input_base_dir}/P{patch}/output/", f"{output_base_dir}/P{patch}" + + +def output_filename(file_pattern, patch, idx): + """Output Filename. + + Build the collated catalogue filename. ``patch`` is None for the patch-less + v2.0 layout, which drops the patch token. + + Parameters + ---------- + file_pattern : str + input file pattern (e.g. ``validation_psf``) + patch : str or None + patch number, or None for the patch-less v2.0 layout + idx : int + exposure run index + + Returns + ------- + str + output catalogue file name + + """ + patch_token = "" if patch is None else f"{patch}-" + return f"{file_pattern}_conv-{patch_token}{idx}.fits" + + +def transform_shape(mom_list, jac): + """Transform Shape. + + Transform shape (ellipticity and size) using a Jacobian. + + Parameters + ---------- + mom_list : list + input moment measurements; each list element contains + first and second ellipticity component and size + jac : galsim.JacobianWCS + Jacobian transformation matrix information + + Returns + ------- + list + transformed shape parameters, which are + first and second ellipticity component and size + + """ + scale, shear, theta, flip = jac.getDecomposition() + + sig_tmp = mom_list[2] * scale + shape = galsim.Shear(g1=mom_list[0], g2=mom_list[1]) + if flip: + # The following output is not observed + print("FLIP!") + shape = galsim.Shear(g1=-shape.g1, g2=shape.g2) + shape = galsim.Shear(g=shape.g, beta=shape.beta + theta) + shape = shear + shape + + return shape.g1, shape.g2, sig_tmp + + +class Loc2Glob(object): + r"""Change from local to global coordinates. + + Class to pass from local coordinates to global coordinates under + CFIS (CFHT) MegaCam instrument. The geometrical informcation of the + instrument is encoded in this function. + + Parameters + ---------- + x_gap : int + Gap between the CCDs along the horizontal direction; + default is ``70`` (MegaCam value) + y_gap : int + Gap between the CCDs along the vertical direction; + Default is ``425`` (MegaCam value) + x_npix : int + Number of pixels per CCD along the horizontal direction; + default is ``2048`` (MegaCam value) + y_npix : int + Number of pixels per CCD along the vertical direction; + default to ``4612`` (MegaCam value) + ccd_tot : int + Total number of CCDs; + default to ``40`` (MegaCam value) + + Notes + ----- + This is the geometry of MegaCam. Watch out with the conventions ba,ab that means where + is the local coordinate system origin for each CCD. + For more info check out MegaCam's instrument webpage. + + Examples + -------- + 'COMMENT (North on top, East to the left)', + 'COMMENT --------------------------', + 'COMMENT ba ba ba ba ba ba ba ba ba', + 'COMMENT 00 01 02 03 04 05 06 07 08', + 'COMMENT --------------------------------', + 'COMMENT ba ba ba ba ba ba ba ba ba ba ba', + 'COMMENT 36 09 10 11 12 13 14 15 16 17 37', + 'COMMENT --------------*-----------------', + 'COMMENT 38 18 19 20 21 22 23 24 25 26 39', + 'COMMENT ab ab ab ab ab ab ab ab ab ab ab', + 'COMMENT --------------------------------', + 'COMMENT 27 28 29 30 31 32 33 34 35', + 'COMMENT ab ab ab ab ab ab ab ab ab', + 'COMMENT __________________________' + """ + + def __init__( + self, x_gap=70, y_gap=425, x_npix=2048, y_npix=4612, ccd_tot=40 + ): + r"""Initialize with instrument geometry.""" + self.x_gap = x_gap + self.y_gap = y_gap + self.x_npix = x_npix + self.y_npix = y_npix + self.ccd_tot = ccd_tot + + def loc2glob_img_coord(self, ccd_n, x_coor, y_coor): + """loc2glob Img Coord. + + Go from the local to the global img (pixel) coordinate system. + + Global system with (0,0) in the intersection of ccds [12,13,21,22]. + + Parameters + ---------- + ccd_n: int + CCD number of the considered positions + x_coor: float + Local coordinate system hotizontal value + y_coor: float + Local coordinate system vertical value + + Returns + ------- + glob_x_coor: float + Horizontal position in global coordinate system + glob_y_coor: float + Vertical position in global coordinate system + + """ + # Flip axes + x_coor, y_coor = self.flip_coord(ccd_n, x_coor, y_coor) + + # Calculate the shift + x_shift, y_shift = self.shift_coord(ccd_n) + + # Return new coordinates + return x_coor + x_shift, y_coor + y_shift + + def flip_coord(self, ccd_n, x_coor, y_coor): + r"""Change of coordinate convention. + + So that all of them are coherent on the global coordinate system. + So that the origin is on the south-west corner. + Positive: South to North ; West to East. + """ + if ccd_n < 18 or ccd_n in [36, 37]: + x_coor = self.x_npix - x_coor + 1 + y_coor = self.y_npix - y_coor + 1 + else: + pass + + return x_coor, y_coor + + def x_coord_range(self): + r"""Return range of the x coordinate.""" + max_x = self.x_npix * 6 + self.x_gap * 5 + min_x = self.x_npix * (-5) + self.x_gap * (-5) + return min_x, max_x + + def y_coord_range(self): + r"""Return range of the y coordinate.""" + max_y = self.y_npix * 2 + self.y_gap * 1 + min_y = self.y_npix * (-2) + self.y_gap * (-2) + return min_y, max_y + + def shift_coord(self, ccd_n): + r"""Provide the shifting. + + It is needed to go from the local coordinate + system origin to the global coordinate system origin. + """ + if ccd_n < 9: + # first row + x_shift = (ccd_n - 4) * (self.x_gap + self.x_npix) + y_shift = self.y_gap + self.y_npix + return x_shift, y_shift + + elif ccd_n < 18: + # second row, non-ears + x_shift = (ccd_n - 13) * (self.x_gap + self.x_npix) + y_shift = 0.0 + return x_shift, y_shift + + elif ccd_n < 27: + # third row non-ears + x_shift = (ccd_n - 22) * (self.x_gap + self.x_npix) + y_shift = -1.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + elif ccd_n < 36: + # fourth row + x_shift = (ccd_n - 31) * (self.x_gap + self.x_npix) + y_shift = -2.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + elif ccd_n < 37: + # ccd= 36 ears, second row + x_shift = (-5.0) * (self.x_gap + self.x_npix) + y_shift = 0.0 + return x_shift, y_shift + + elif ccd_n < 38: + # ccd= 37 ears, second row + x_shift = 5.0 * (self.x_gap + self.x_npix) + y_shift = 0.0 + return x_shift, y_shift + + elif ccd_n < 39: + # ccd= 38 ears, third row + x_shift = (-5.0) * (self.x_gap + self.x_npix) + y_shift = -1.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + elif ccd_n < 40: + # ccd= 39 ears, third row + x_shift = 5.0 * (self.x_gap + self.x_npix) + y_shift = -1.0 * (self.y_gap + self.y_npix) + return x_shift, y_shift + + +class Glob2CCD(object): + r"""Get the CCD ID number from the global coordinate position. + + The Loc2Glob() object as input is the one that defines the instrument's + geometry. + + Parameters + ---------- + loc2glob: Loc2Glob object + Object with the desired focal plane geometry. + with_gaps: bool + If add the gaps to the CCD area. + """ + + def __init__(self, loc2glob, with_gaps=True): + # Save loc2glob object + self.loc2glob = loc2glob + self.with_gaps = with_gaps + self.ccd_list = np.arange(self.loc2glob.ccd_tot) + # Init edges defininf the CCDs + self.edge_x_list, self.edge_y_list = self.build_all_edges() + + def build_all_edges(self): + """Build the edges for all the CCDs in the focal plane.""" + edge_xy_list = [] + for idx in (0, 1): + edge_list = np.array( + [self.build_edge(ccd_n)[idx] for ccd_n in self.ccd_list] + ) + edge_xy_list.append(edge_list) + + return edge_xy_list + + def build_edge(self, ccd_n): + """Build the edges of the `ccd_n` in global coordinates.""" + if self.with_gaps: + corners = np.array( + [ + [-self.loc2glob.x_gap / 2, -self.loc2glob.y_gap / 2], + [ + self.loc2glob.x_npix + self.loc2glob.x_gap / 2, + -self.loc2glob.y_gap / 2, + ], + [ + -self.loc2glob.x_gap / 2, + self.loc2glob.y_npix + self.loc2glob.y_gap / 2, + ], + [ + self.loc2glob.x_npix + self.loc2glob.x_gap / 2, + self.loc2glob.y_npix + self.loc2glob.y_gap / 2, + ], + ] + ) + else: + corners = np.array( + [ + [0, 0], + [self.loc2glob.x_npix, 0], + [0, self.loc2glob.y_npix], + [self.loc2glob.x_npix, self.loc2glob.y_npix], + ] + ) + + glob_corners = np.array( + [ + self.loc2glob.loc2glob_img_coord(ccd_n, pos[0], pos[1]) + for pos in corners + ] + ) + + edge_xy = [] + for idx in (0, 1): + edge = np.array( + [np.min(glob_corners[:, idx]), np.max(glob_corners[:, idx])] + ) + edge_xy.append(edge) + + return edge_xy + + def is_inside(self, x, y, edge_x, edge_y): + """Is the position inside the edges. + + Return True if the position is within the rectangle + defined by the edges. + + Parameters + ---------- + x: float + Horizontal position in global coordinate system. + y: float + Vertical position in global coordinate system. + edge_x: np.ndarray + Edge defined as `np.array([min_x, max_x])`. + edge_y: np.ndarray + Edge defined as `np.array([min_y, max_y])`. + """ + if ( + (x > edge_x[0]) + and (x < edge_x[1]) + and (y > edge_y[0]) + and (y < edge_y[1]) + ): + return True + else: + return False + + def get_ccd_n(self, x, y): + """Returns the CCD number from the position `(x, y)`. + + Returns `None` if the position is not found. + """ + bool_list = np.array( + [ + self.is_inside(x, y, edge_x, edge_y) + for edge_x, edge_y in zip(self.edge_x_list, self.edge_y_list) + ] + ) + + try: + return self.ccd_list[bool_list][0] + except Exception: + return None + + +class Convert(object): + + def __init__(self): + + self.params_default() + + def set_params_from_command_line(self, args): + """Set Params From Command line. + + Only use when calling using python from command line. + Does not work from ipython or jupyter. + + """ + # Read command line options + options = cs_args.parse_options( + self._params, + self._short_options, + self._types, + self._help_strings, + ) + self._params = options + + # Save calling command + logging.log_command(args) + + def params_default(self): + + self._params = { + "input_base_dir": ".", + "output_base_dir": ".", + "version_cat": "v2.0", + "mode": "merge", + "patches": "", + "psf": "psfex", + "file_pattern_psfint": "validation_psf", + } + + self._short_options = { + "input_base_dir": "-i", + "version_cat": "-V", + "mode": "-m", + "psf": "-p", + "patches": "-P", + } + + self._types = {} + + self._help_strings = { + "input_base_dir": ( + "input base dir; for v1.x runs are expected in" + + " /P/output, for v2.0 (patch-less) in" + + " /output; default is {}" + ), + "version_cat": ( + "catalogue major version, allowed are v1.3, v1.4, v1.5, v1.6," + + " v2.0; v2.0 is patch-less; default is {}" + ), + "mode": ( + "run mode, allowed are 'merge', 'test'; default is" + " '{}'" + ), + "psf": "PSF model, allowed are 'psfex' and 'mccd'; default is {}", + "patches": "(list of) input patches; ignored for v2.0", + } + + # Output column names with types + self._dt = [ + ("X", float), + ("Y", float), + ("RA", float), + ("DEC", float), + ("E1_PSF_HSM", float), + ("E2_PSF_HSM", float), + ("SIGMA_PSF_HSM", float), + ("FLAG_PSF_HSM", float), + ("E1_STAR_HSM", float), + ("E2_STAR_HSM", float), + ("SIGMA_STAR_HSM", float), + ("FLAG_STAR_HSM", float), + ("CCD_NB", int), + ] + + # Extra columns for MCCD:737 + self._dt_mccd = self._dt.copy() + self._dt_mccd.append(("GLOB_X", float)) + self._dt_mccd.append(("GLOB_Y", float)) + + def update_params(self): + """Update Params. + + Update parameters. + + """ + if self._params["psf"] == "psfex": + #self._params["sub_dir_pattern"] = "run_sp_exp_202" + self._params["sub_dir_pattern"] = "run_sp_combined_psf" + self._params["sub_dir_psfint"] = "psfex_interp_runner" + elif self._params["psf"] == "mccd": + self._params["sub_dir_pattern"] = "run_sp_exp_SxSePsf_202" + self._params["sub_dir_psfint"] = "mccd_fit_val_runner" + self._params["sub_dir_setools"] = "setools_runner/output/mask" + else: + raise ValueError(f"Invalid PSF model {self._params['psf']}") + self._params["sub_dir_psfint"] = ( + f"{self._params['sub_dir_psfint']}/output" + ) + + def run(self): + """Run. + + Main processing function. + + """ + # Guard against a mistyped version silently falling through to the + # v1.x patch loop (e.g. ``-V v2`` or ``-V 2.0``). + allowed_versions = ("v1.3", "v1.4", "v1.5", "v1.6", "v2.0") + if self._params["version_cat"] not in allowed_versions: + raise ValueError( + f"Invalid version {self._params['version_cat']}; allowed are" + + f" {', '.join(allowed_versions)}" + ) + + # v2.0 removes the patch concept: a single patch-less run root. For + # v1.x, iterate over the requested sky patches as before. ``patch`` is + # None in the patch-less case, which drops the patch token from the + # input path and the output filename. + if self._params["version_cat"] == "v2.0": + patch_nums = [None] + elif self._params["mode"] == "test": + patch_nums = ["3", "4"] + else: + patch_nums = cs_args.my_string_split(self._params["patches"]) + + do_parallel = True + + # Loop over patches + for patch in patch_nums: + + patch_dir, output_dir = collate_paths( + self._params["input_base_dir"], + self._params["output_base_dir"], + patch, + ) + print("Running patch-less (v2.0)" if patch is None else f"Running patch: {patch}") + + if not os.path.isdir(output_dir): + os.makedirs(output_dir, exist_ok=True) + + subdirs = f"{patch_dir}/{self._params['sub_dir_pattern']}*" + exp_run_dirs = glob.glob(subdirs) + n_exp_runs = len(exp_run_dirs) + print( + f"Found {n_exp_runs} input single-exposure run(s) for patch" + + f" {patch_dir} ({subdirs})" + ) + + if self._params["mode"] == "test": + exp_run_dirs = exp_run_dirs[:2] + n_exp_runs = len(exp_run_dirs) + print( + f"test mode: only using {n_exp_runs} input single-exposure" + + f" runs" + ) + + # Loop over exposure runs + if not do_parallel: + for idx_exp, exp_run_dir in tqdm( + enumerate(exp_run_dirs), + total=n_exp_runs, + disable=self._params["verbose"], + ): + self.transform_exposures( + output_dir, patch, idx_exp, exp_run_dir + ) + else: + res = Parallel(n_jobs=-1, backend="loky")( + delayed(self.transform_exposures)( + output_dir, patch, idx_exp, exp_run_dir + ) + for idx_exp, exp_run_dir in tqdm( + enumerate(exp_run_dirs), + total=n_exp_runs, + disable=self._params["verbose"], + ) + ) + + def transform_exposures(self, output_dir, patch, idx, exp_run_dir): + """Transform exposures. + + Transform shapes for exposure for a given run (input exp run dir). + + """ + output_path = ( + f"{output_dir}/" + + output_filename( + self._params["file_pattern_psfint"], patch, idx + ) + ) + if os.path.exists(output_path): + print(f"Skipping transform_exposures, file {output_path} exists") + return + + psf_dir = f"{exp_run_dir}/{self._params['sub_dir_psfint']}" + try: + all_files = os.listdir(psf_dir) + if self._params["verbose"]: + print(f"Found {len(all_files)} file(s) in {psf_dir}") + except Exception: + if self._params["verbose"]: + print(f"Found zero PSFEx files in {psf_dir}, skipping") + return + + cat_list = [] + for file_name in all_files: + if self._params["file_pattern_psfint"] not in file_name: + continue + + tmp = re.findall(r"\d+", file_name) + + if self._params["psf"] == "psfex": + exp_name, ccd_id = int(tmp[0]), int(tmp[1]) + elif self._params["psf"] == "mccd": + exp_name = int(tmp[0]) + ccd_id = -1 + + if self._params["verbose"]: + print("Match found ", exp_name, ccd_id) + + psf_file_path = f"{psf_dir}/{file_name}" + + try: + if self._params["psf"] == "psfex": + psf_file_hdus = fits.open(psf_file_path, memmap=False) + psf_file = psf_file_hdus[2].data + psf_file_hdus.close() + mod = "RA" + else: + psf_file = fits.getdata(psf_file_path, 1, memmap=True) + mod = "RA_LIST" + except Exception: + continue + + if self._params["psf"] == "psfex": + # HSM ellipticities and sizes are measured directly in world + # coordinates upstream (FindAdaptiveMom use_sky_coords=True), so + # they are passed straight through; only positions are collated. + exp_cat = np.array( + list( + map( + tuple, + np.array( + [ + psf_file["X"], + psf_file["Y"], + psf_file["RA"], + psf_file["DEC"], + psf_file["E1_PSF_HSM"], + psf_file["E2_PSF_HSM"], + psf_file["SIGMA_PSF_HSM"], + psf_file["FLAG_PSF_HSM"], + psf_file["E1_STAR_HSM"], + psf_file["E2_STAR_HSM"], + psf_file["SIGMA_STAR_HSM"], + psf_file["FLAG_STAR_HSM"], + np.ones_like(psf_file["RA"], dtype=int) + * ccd_id, + ] + ).T.tolist(), + ) + ), + dtype=self._dt, + ) + cat_list.append(exp_cat) + + else: + l2g = Loc2Glob() + g2c = Glob2CCD(l2g) + new_ccd_id = np.array( + [ + int( + g2c.get_ccd_n( + psf_file["GLOB_POSITION_IMG_LIST"][ii, 0], + psf_file["GLOB_POSITION_IMG_LIST"][ii, 1], + ) + ) + for ii in range(len(psf_file)) + ] + ) + + # Local-to-CCD position: subtract each CCD's focal-plane shift. + new_x = np.zeros_like(psf_file[mod]) + new_y = np.zeros_like(psf_file[mod]) + + # The MCCD PSF_MOM_LIST/STAR_MOM_LIST columns come from the + # external mccd fit-validation code, which still measures HSM + # moments in the pixel frame; rotate them into world coordinates + # via the per-CCD WCS Jacobian. This rotation stays until mccd + # itself adopts use_sky_coords (see the module docstring). The + # in-repo PSFEx / MCCD-interpolation paths are already in world + # coordinates and are passed through unrotated. + new_e1_psf = np.zeros_like(psf_file[mod]) + new_e2_psf = np.zeros_like(psf_file[mod]) + new_sig_psf = np.zeros_like(psf_file[mod]) + new_e1_star = np.zeros_like(psf_file[mod]) + new_e2_star = np.zeros_like(psf_file[mod]) + new_sig_star = np.zeros_like(psf_file[mod]) + new_flag_psf = np.zeros_like(psf_file[mod]) + new_flag_star = np.zeros_like(psf_file[mod]) + for ccd_id in range(40): + m_ccd_id = new_ccd_id == ccd_id + if sum(m_ccd_id) == 0: + continue + + x_shift, y_shift = l2g.shift_coord(ccd_id) + + new_x[m_ccd_id] = ( + psf_file["GLOB_POSITION_IMG_LIST"][:, 0][m_ccd_id] + - x_shift + ) + new_y[m_ccd_id] = ( + psf_file["GLOB_POSITION_IMG_LIST"][:, 1][m_ccd_id] + - y_shift + ) + + header_file_path = ( + self._params["sub_dir_setools"] + + self._params["file_pattern_psfint"] + + f"{exp_name}-{ccd_id}.fits" + ) + try: + header_file = fits.getdata(header_file_path, 1) + except Exception: + continue + header = fits.Header.fromstring( + "\n".join(header_file[0][0]), sep="\n" + ) + wcs = galsim.AstropyWCS(header=header) + + g1_psf_tmp_l = [] + g2_psf_tmp_l = [] + sig_psf_tmp_l = [] + g1_star_tmp_l = [] + g2_star_tmp_l = [] + sig_star_tmp_l = [] + flag_psf_tmp_l = [] + flag_star_tmp_l = [] + + for obj in psf_file[m_ccd_id]: + try: + jac = wcs.jacobian( + world_pos=galsim.CelestialCoord( + ra=obj["RA_LIST"] * galsim.degrees, + dec=obj["DEC_LIST"] * galsim.degrees, + ) + ) + except Exception: + flag_star_tmp_l.append(16) + flag_psf_tmp_l.append(16) + g1_psf_tmp_l.append(0) + g2_psf_tmp_l.append(0) + sig_psf_tmp_l.append(0) + g1_star_tmp_l.append(0) + g2_star_tmp_l.append(0) + sig_star_tmp_l.append(0) + continue + g1_psf_tmp, g2_psf_tmp, sig_psf_tmp = transform_shape( + obj["PSF_MOM_LIST"], jac + ) + + g1_psf_tmp_l.append(g1_psf_tmp) + g2_psf_tmp_l.append(g2_psf_tmp) + sig_psf_tmp_l.append(sig_psf_tmp) + flag_psf_tmp_l.append(obj["PSF_MOM_LIST"][3]) + + g1_star_tmp, g2_star_tmp, sig_star_tmp = ( + transform_shape(obj["STAR_MOM_LIST"], jac) + ) + g1_star_tmp_l.append(g1_star_tmp) + g2_star_tmp_l.append(g2_star_tmp) + sig_star_tmp_l.append(sig_star_tmp) + flag_star_tmp_l.append(obj["STAR_MOM_LIST"][3]) + + new_e1_psf[m_ccd_id] = g1_psf_tmp_l + new_e2_psf[m_ccd_id] = g2_psf_tmp_l + new_sig_psf[m_ccd_id] = sig_psf_tmp_l + new_flag_psf[m_ccd_id] = flag_psf_tmp_l + new_e1_star[m_ccd_id] = g1_star_tmp_l + new_e2_star[m_ccd_id] = g2_star_tmp_l + new_sig_star[m_ccd_id] = sig_star_tmp_l + new_flag_star[m_ccd_id] = flag_star_tmp_l + + exp_cat = np.array( + list( + map( + tuple, + np.array( + [ + new_x, + new_y, + psf_file["RA_LIST"], + psf_file["DEC_LIST"], + new_e1_psf, + new_e2_psf, + new_sig_psf, + psf_file["PSF_MOM_LIST"][:, 3], + new_e1_star, + new_e2_star, + new_sig_star, + psf_file["STAR_MOM_LIST"][:, 3], + new_ccd_id, + psf_file["GLOB_POSITION_IMG_LIST"][:, 0], + psf_file["GLOB_POSITION_IMG_LIST"][:, 1], + ] + ).T.tolist(), + ) + ), + dtype=self._dt_mccd, + ) + cat_list.append(exp_cat) + + del psf_file + + if len(cat_list) == 0: + return + + # Finalize catalogue + patch_cat = np.concatenate(cat_list) + hdul = fits.HDUList() + hdul.append(fits.PrimaryHDU()) + hdul.append(fits.BinTableHDU(patch_cat)) + + # Write catalogue + hdul.writeto( + output_path, + overwrite=True, + ) + + del cat_list + del hdul + gc.collect() + + +def run_convert(*args): + + # Create instance + obj = Convert() + + obj.set_params_from_command_line(args) + obj.update_params() + + obj.run() + + +def main(argv=None): + """Main + + Main program + """ + if argv is None: + argv = sys.argv[1:] + run_convert(*argv) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/module/test_collate_star_cat.py b/tests/module/test_collate_star_cat.py new file mode 100644 index 000000000..20d52ce23 --- /dev/null +++ b/tests/module/test_collate_star_cat.py @@ -0,0 +1,70 @@ +"""UNIT TESTS FOR STAR-CATALOGUE COLLATION PATHS. + +Pin the patch vs patch-less (v2.0) path and filename convention of +``scripts/python/collate_star_cat.py``. Runs up to v1.6 carry a ``P`` +token in both the input run directory and the output filename; v2.0 is +patch-less (``patch is None``) and drops that token, reading from a single +``/output`` root and writing ``validation_psf_conv-.fits`` — the +name still matched by the downstream ``validation_psf_conv-*`` glob. +""" + +import importlib.util +from pathlib import Path + +import pytest + +# The collation script lives under scripts/python (not an importable package), +# so load it by path. +_SCRIPT = ( + Path(__file__).resolve().parents[2] + / "scripts" + / "python" + / "collate_star_cat.py" +) +_spec = importlib.util.spec_from_file_location("collate_star_cat", _SCRIPT) +collate_star_cat = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(collate_star_cat) + + +@pytest.mark.parametrize( + "patch, exp_input, exp_output", + [ + ("3", "in/P3/output/", "out/P3"), + (None, "in/output/", "out"), + ], +) +def test_collate_paths(patch, exp_input, exp_output): + """v1.x carries the P token; v2.0 (patch None) drops it.""" + assert collate_star_cat.collate_paths("in", "out", patch) == ( + exp_input, + exp_output, + ) + + +@pytest.mark.parametrize( + "patch, expected", + [ + ("3", "validation_psf_conv-3-0.fits"), + (None, "validation_psf_conv-0.fits"), + ], +) +def test_output_filename(patch, expected): + """The patch token is present for v1.x and absent for v2.0.""" + assert collate_star_cat.output_filename("validation_psf", patch, 0) == expected + + +def test_output_filename_matches_downstream_glob(): + """Both layouts stay under the downstream ``validation_psf_conv-*`` glob.""" + for patch in ("1", None): + assert collate_star_cat.output_filename( + "validation_psf", patch, 5 + ).startswith("validation_psf_conv-") + + +@pytest.mark.parametrize("bad", ["v2", "2.0", "v1.7", ""]) +def test_invalid_version_raises(bad): + """A mistyped -V is rejected rather than falling through to v1.x.""" + obj = collate_star_cat.Convert() + obj._params["version_cat"] = bad + with pytest.raises(ValueError): + obj.run() From ed376f734cc9dd46c7c2f93a67b2180f397d110a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:07:24 -0400 Subject: [PATCH 14/17] fix(mask_query): public sentinel, empty-coverage guard, real product name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three from final review. `mask_map.sentinel` replaces `mask_map._sentinel` in the probe path — same value (verified for both int and bool maps on healsparse 1.12.2), public API. The probe also indexed `np.flatnonzero(coverage.coverage_mask)[0]` unguarded, so a map with no coverage at all died with a bare IndexError from inside the utility. It now raises `ValueError("healsparse map has empty coverage")`, naming the file, with a test. The shipped MASK_PATHS placeholder becomes the real DR6 product name, `mask_r_nside131072_n4.hsp`, in all three configs. The products are staged at /project/6001537/cdaley/masks/dr6/ (same tree as /project/def-mjhudson/cdaley/masks/dr6/), which is where the partial-read measurement in f7d1fd66 was taken — that docstring already named the file correctly, so nothing else mentioned the old placeholder. 25 tests pass in the container (mask_query, make_cat_mask_ext, collate_star_cat); 65 configs resolve. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_exp_mccd.ini | 2 +- example/cfis/config_exp_psfex.ini | 2 +- src/shapepipe/utilities/mask_query.py | 10 +++++++--- tests/module/test_mask_query.py | 11 +++++++++++ workflow/config/cfis/config_exp_psfex.ini | 2 +- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 68dcfbeef..5d2460312 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -145,7 +145,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 2f1c8e353..b4e2d1fa1 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -147,7 +147,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index c3b150006..d789d4100 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -169,9 +169,13 @@ def query_map_coverage(path, ra, dec): # the empty case is answered without asking it: load one arbitrary # coverage pixel purely to learn the dtype and sentinel, and return # that sentinel everywhere. Same answer, one small read. - probe = int(np.flatnonzero(coverage.coverage_mask)[0]) - mask_map = healsparse.HealSparseMap.read(path, pixels=[probe]) - values = np.full(ra.size, mask_map._sentinel, dtype=mask_map.dtype) + covered = np.flatnonzero(coverage.coverage_mask) + if covered.size == 0: + raise ValueError(f"healsparse map {path} has empty coverage") + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(covered[0])] + ) + values = np.full(ra.size, mask_map.sentinel, dtype=mask_map.dtype) return values, in_coverage mask_map = healsparse.HealSparseMap.read( diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py index 779a6bd69..85ce028cd 100644 --- a/tests/module/test_mask_query.py +++ b/tests/module/test_mask_query.py @@ -299,3 +299,14 @@ def test_mask_query_empty_ccd(tmp_path): assert flag.shape == (0,) assert "FLAG_EXT" in names assert any("No detections" in m for m in log.info_msgs) + + +def test_empty_coverage_raises_clearly(tmp_path): + """A map with no coverage at all fails with a message naming the file. + + The probe read in query_map_coverage indexes the first covered pixel; with + nothing covered that would be an IndexError from deep inside the utility. + """ + path = _write_map(tmp_path / "empty.hsp", 4, n_covered=0) + with pytest.raises(ValueError): + mask_query_util.query_map_coverage(path, RA, DEC) diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index ffcac86b6..2fa5dc525 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -146,7 +146,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map From c7c9e82f3d69d4a93393518d847f3e48243c2692 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:33:52 -0400 Subject: [PATCH 15/17] =?UTF-8?q?refactor(random=5Fcat):=20delete=20?= =?UTF-8?q?=E2=80=94=20randoms=20come=20from=20healsparse=20map=20algebra?= =?UTF-8?q?=20downstream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cail's call. random_cat computed a tile's effective unmasked area and drew randoms inside it by counting zero pixels in a tile mask IMAGE. That input was the deleted mask module's output, and with the query design there is no such image and no plausible producer for one: the survey window is map algebra on the healsparse coverage map (#797), done downstream, not a per-tile pipeline step that rasterizes to pixels first. Rather than keep a module wired to a placeholder path nobody can fill — which is what f7d1fd66 left, an honest but unpaid IOU — the module goes: the package, the runner, config_Rc.ini, docs/source/random_cat.md and its toc.rst entry. There were no tests to remove. The module was already unsound independently of masking, which is part of why keeping it had no value: `save_as_healpix` is called as `_save_as_healpix`, and `process` references `file_name` and `output_dir` which are never bound. Any run past the first few lines would have raised. `reproject` leaves pyproject.toml with it — random_cat.py held its only import (the two ngmix hits for "reprojection" are prose in comments). uv.lock regenerated: reproject, plus pims, pyavm, slicerator, toolz and zarr that came in behind it. Verified in the container: 29 runners import (was 30), 64 configs resolve (was 65, 0 bad), the 25 mask/collate tests pass, the workflow still builds its DAG, and random_cat / N_RANDOM / RandomCat / config_Rc / run_sp_Rc appear nowhere outside scripts/sh/, which stays untouched as agreed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/random_cat.md | 105 -------- docs/source/toc.rst | 1 - example/cfis/config_Rc.ini | 79 ------ pyproject.toml | 1 - .../modules/random_cat_package/__init__.py | 37 --- .../modules/random_cat_package/random_cat.py | 235 ------------------ src/shapepipe/modules/random_cat_runner.py | 81 ------ uv.lock | 222 ----------------- 8 files changed, 761 deletions(-) delete mode 100644 docs/source/random_cat.md delete mode 100644 example/cfis/config_Rc.ini delete mode 100644 src/shapepipe/modules/random_cat_package/__init__.py delete mode 100644 src/shapepipe/modules/random_cat_package/random_cat.py delete mode 100644 src/shapepipe/modules/random_cat_runner.py diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md deleted file mode 100644 index b5ae9931d..000000000 --- a/docs/source/random_cat.md +++ /dev/null @@ -1,105 +0,0 @@ -# Create random catalogues and masks - -This section describes how to create tile-based random catalogues and healpix -masks, and combined randoms and masks for a selection of tiles. - -The masked regions are obtained on input from per-tile pixel mask images. - -```{warning} -**ShapePipe no longer produces those images.** The pipeline generates no masks -at all: sky-fixed masks are healsparse maps, queried once per object into -catalogue columns (see [Masks](pipeline_tutorial.md#masks)), and tiles have no -flag image. `random_cat_runner` therefore needs its mask images supplied from -outside the pipeline — point its second `INPUT_DIR` entry at a directory of tile -mask images matching its `NUMBERING_SCHEME`. The healsparse-native replacement -for this whole procedure (an n_epoch / n_pointings survey-window map built from -the maps directly) is issue #797. -``` - -```{note} -Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see -[VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The input-staging and joint-mask steps now -overlap with [`sp_validation`](https://github.com/CosmoStat/sp_validation). The -steps are retained for reference. -``` - -## Set up - -### ID file and shell variables - -First, if if does not exist already, create the file ``tile_numbers.txt`` containing a list of tile IDs, -one per line. This is the same format as the input file to ``get_images_runner``. -For example, link to a patch ID list, -```bash -ln -s tiles_PX.txt tile_numbers.txt -``` -Next, set the run and config paths, -```bash -export SP_RUN=. -export SP_CONFIG=/path/to/config-files -``` - -### Get images or image headers - -We need to footprint of the image tiles. If they have been downloaded for a ``ShapePipe`` run, -check that they are accessible as last run of the ``get_images_runner`` module. - -If not, we can just download the headers to gain significant download time. -```bash -shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini -``` - -### Stage the pixel mask files - -Collect the tile mask images into one directory that `random_cat_runner`'s -`INPUT_DIR` points at, one file per tile ID in `tile_numbers.txt`, named to -match the module's `FILE_PATTERN` and `NUMBERING_SCHEME` (`mask--.fits` -with the committed `config_Rc.ini`). How you obtain them is outside ShapePipe; -older runs of the pipeline's own (now removed) mask module wrote them as -`pipeline_flag--.fits`, and those files still work. - -## Create random catalogue and helapix mask per tile - -Run -```bash -shapepipe_run -c $SP_CONFIG/config_Rc.ini -``` -The random catalogue and, with the config entry ``SAVE_MASK_AS_HEALPIX = True`` -a healpix mask FITS file, will be written to disk. - -## Create joint random catalogue - -The individual tile-based random catalogues can be merged into a numpy -binary (``.npy``) file with -```bash -merge_final_cat -i output/run_sp_Rc/random_cat_runner/output -n random_cat -v -``` - -### Results - -We can plot the random objects, -```bash -python ~/astro/repositories/github/sp_validation/scripts/plot_rand.py -``` -and also compute the effective survey area, -```bash -~/astro/repositories/github/sp_validation/scripts/compute_area.py -``` - -## Create joint healpix mask - -First, for convenience all image headers with WCS information are -linked from within one directory, with -```bash -prepare_tiles_for_final -i -``` - -Next, read all tile mask and WCS information, and create a joint full-sky -healpix mask with -```bash -/path/to/sp_validation/scripts/scripts/combine_hp_masks.py -p -v -``` -With the option ``-p`` the mask is plotted in Mollweid projection. - - diff --git a/docs/source/toc.rst b/docs/source/toc.rst index c08c4773c..6724b2b42 100644 --- a/docs/source/toc.rst +++ b/docs/source/toc.rst @@ -41,7 +41,6 @@ :caption: Miscellaneous post_processing - random_cat .. toctree:: :hidden: diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini deleted file mode 100644 index 9f55f1812..000000000 --- a/example/cfis/config_Rc.ini +++ /dev/null @@ -1,79 +0,0 @@ -# ShapePipe configuration file for: create random catalogue - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_Rc - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = random_cat_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 24 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options -[RANDOM_CAT_RUNNER] - -# The mask image is now an EXTERNAL product: ShapePipe generates no tile -# masks. Point the second entry at a directory of tile mask images matching -# NUMBERING_SCHEME below (the healsparse-native replacement for this module is -# the survey-window work, issue #797). -INPUT_DIR = last:get_images_runner, - -FILE_PATTERN = CFIS_image, mask - -NUMBERING_SCHEME = 000-000 - -# Number of random objects -N_RANDOM = 50000 - -# N_RANDOM is per square degrees if True -DENSITY = True - -# Output healpix mask if True -SAVE_MASK_AS_HEALPIX = True - -# Healpix mask file base name (used if SAVE_MASK_AS_HEALPIX is True) -HEALPIX_OUT_FILE_BASE = mask_hp - -# Healpix mask nside (used if SAVE_MASK_AS_HEALPIX is True) -HEALPIX_OUT_NSIDE = 1024 diff --git a/pyproject.toml b/pyproject.toml index 4c5fd7e02..47ce2b2d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ dependencies = [ "python-pysap>=0.3", "PyQt5", "pyqtgraph", - "reproject>=0.19", "sf_tools>=2.0.4", "skaha>=1.7", "sqlitedict>=2.0", diff --git a/src/shapepipe/modules/random_cat_package/__init__.py b/src/shapepipe/modules/random_cat_package/__init__.py deleted file mode 100644 index 32c6f64b2..000000000 --- a/src/shapepipe/modules/random_cat_package/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""RANDOM CATALOGUE PACKAGE. - -This package contains the module for ``random_cat``. - -:Author: Martin Kilbinger - -:Parent module: None - -:Input: Images and masks - -:Output: Random catalogue FITS file - -Description -=========== - -This module creates a random catalogue, and computes the tile area accounting -for overlapping and masked regions. - -Module-specific config file entries -=================================== - -N_RANDOM : float - The number of random objects requested on output -DENSITY : bool, optional - Option to interpret the number of random objects per square degree; the - default is ``False`` -SAVE_MASK_AS_HEALPIX : bool - Output healpix mask if ``True`` -HEALPIX_OUT_FILE_BASE : str, optional - Output halpix mask file base name; used only if SAVE_MASK_AS_HEALPIX is - ``True`` -HEALPIX_OUT_NSIDE : int, optional - Output healpix mask nside; used only if SAVE_MASK_AS_HEALPIX is ``True`` - -""" - -__all__ = ["random_cat.py"] diff --git a/src/shapepipe/modules/random_cat_package/random_cat.py b/src/shapepipe/modules/random_cat_package/random_cat.py deleted file mode 100644 index cabbd18e0..000000000 --- a/src/shapepipe/modules/random_cat_package/random_cat.py +++ /dev/null @@ -1,235 +0,0 @@ -"""RANDOM CATALOGUE. - -This module contains a class to create a random catalogue, and to compute -the tile area accounting for overlapping and masked regions. - -:Author: Martin Kilbinger - -""" - -import os -import re - -import numpy as np - -import astropy.io.fits as fits -from astropy import wcs -from astropy.table import Table - -from reproject import reproject_to_healpix - -from shapepipe.pipeline import file_io -from shapepipe.utilities import cfis - - -class RandomCat: - """Random Catalogue. - - This class creates a random catalogue given a mask FITS file. - - Parameters - ---------- - input_image_path : str - Path to input image file - input_mask_path : str - Path to input mask file - output_dir : str - Output directory - file_number_pattern : str - ShapePipe image ID string - output_file_pattern : str - Output file pattern (base name) for random catalogue - n_rand : float - Number of random objects on output - density : bool - ``n_rand`` is interpreted per square degrees if ``True`` - w_log : logging.Logger - Logging instance - healpix_options : dict - Parameters for HEALPix output mask file - """ - - def __init__( - self, - input_image_path, - input_mask_path, - output_dir, - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ): - - self._input_image_path = input_image_path - self._input_mask_path = input_mask_path - self._output_dir = output_dir - self._file_number_string = file_number_string - self._output_file_pattern = output_file_pattern - self._n_rand = n_rand - self._density = density - self._w_log = w_log - self._healpix_options = healpix_options - - def save_as_healpix(self, hdu_mask, header): - """Save As Healpix. - - Save mask as healpix FITS file. - - Parameters - ---------- - hdu_mask : class HDUList - HDU with 2D pixel mask image - header : class Header - Image header with WCS information - - """ - if not self._healpix_options: - return - - mask_1d, footprint = reproject_to_healpix( - (hdu_mask, header), - 'galactic', - nside=self._healpix_options['OUT_NSIDE'] - ) - - t = Table() - t['flux'] = mask_1d - t.meta['ORDERING'] = 'RING' - t.meta['COORDSYS'] = 'G' - t.meta['NSIDE'] = self._healpix_options['OUT_NSIDE'] - t.meta['INDXSCHM'] = 'IMPLICIT' - - output_path = ( - f'{output_dir}/{self._healpix_options["FILE_BASE"]}-' - + f'{file_number_string}.fits' - ) - t.write(output_path) - - def process(self): - """Process. - - Main function to identify exposures. - - """ - # Read image FITS file header - try: - img = fits.open(self._input_image_path) - header = img[0].header - except (OSError, IOError) as error: - # FITS file might contain only header. - # Try as ascii file - try: - fin = open(self._input_image_path) - header = fits.Header.fromtextfile(fin) - fin.close() - except Exception: - raise - - # Get WCS - WCS = wcs.WCS(header) - - # Read mask FITS file - hdu_mask = fits.open(self._input_mask_path) - mask = hdu_mask[0].data - - # Save mask in healpix format (if option is set) - self._save_as_healpix(hdu_mask, header) - - # Number of pixels - n_pix_x = mask.data.shape[0] - n_pix_y = mask.data.shape[1] - n_pix = n_pix_x * n_pix_y - - # Number of non-masked pixels - n_unmasked = len(np.where(mask == 0)[0]) - - # Compute various areas - - # Pixel area in deg^2 - area_pix = wcs.utils.proj_plane_pixel_area(WCS) - - # Tile area - area_deg2 = area_pix * n_pix - - # Area of unmasked region - area_deg2_eff = area_pix * n_unmasked - - # Compute number of requested objects - if n_unmasked > 0: - if not self._density: - # Use value from config file - n_obj = self._n_rand - else: - # Compute number of objects from density - n_obj = int( - self._n_rand / area_deg2 * area_deg2_eff / area_deg2 - ) - - # Check that a reasonably large number of pixels is not masked - if n_unmasked < n_obj: - raise ValueError( - f"Number of un-masked pixels {n_unmasked} is smaller " - + f"than number of random objects requested {n_obj}" - ) - - else: - n_obj = 0 - - self._w_log.info(f"Creating {n_obj} random objects") - - # Draw points until n are in mask - n_found = 0 - xy_rand = [] - while n_found < n_obj: - idx_x = np.random.randint(n_pix_x) - idx_y = np.random.randint(n_pix_y) - - # Add points with additional random sub-pixel value - if mask[idx_x, idx_y] == 0: - d = np.random.random(2) - # MKDEBUG: the following seems to work, x and y interchanged - xy_rand.append([idx_y + d[1], idx_x + d[0]]) - n_found = n_found + 1 - xy_rand = np.array(xy_rand) - - # Transform to WCS - res = WCS.all_pix2world(xy_rand, 1) - if n_unmasked > 0: - ra_rand = res[:, 0] - dec_rand = res[:, 1] - x_rand = xy_rand[:, 0] - y_rand = xy_rand[:, 1] - else: - ra_rand = [] - dec_rand = [] - x_rand = [] - y_rand = [] - - # Tile ID - output_path = ( - f"{self._output_dir}/{self._output_file_pattern}-" - + f"{self._file_number_string}.fits" - ) - file_base = os.path.splitext(file_name)[0] - tile_ID_str = re.split("-", file_base)[1:] - tile_id = float(".".join(tile_ID_str)) - tile_id_array = np.ones(n_obj) * tile_id - - # Write to output - cat_out = [ra_rand, dec_rand, x_rand, y_rand, tile_id_array] - column_names = ["RA", "DEC", "x", "y", "TILE_ID"] - - # TODO: Add units to header - output = file_io.FITSCatalogue( - output_path, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite - ) - output.save_as_fits(cat_out, names=column_names) - - # Write area information to log file - self._w_log.info(f"Total area = {area_deg2:.4f} deg^2") - self._w_log.info(f"Unmasked area = {area_deg2_eff:.4f} deg^2") - self._w_log.info( - f"Ratio masked to total pixels = {n_unmasked / n_pix:.3f}" - ) diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py deleted file mode 100644 index f180c9f7e..000000000 --- a/src/shapepipe/modules/random_cat_runner.py +++ /dev/null @@ -1,81 +0,0 @@ -"""RANDOM CAT RUNNER. - -Module runner for ``random_cat``. - -:Author: Martin Kilbinger - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.random_cat_package.random_cat import RandomCat - - -@module_runner( - version="1.1", - # The mask image is an external input: ShapePipe generates no tile masks - # (the healsparse-native replacement for this module is issue #797). - file_pattern=["image", "mask"], - file_ext=[".fits", "fits"], - depends=["astropy"], - numbering_scheme="_0", -) -def random_cat_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Random Catalogue Runner.""" - # Get input file names of image and mask - input_image_name = input_file_list[0] - input_mask_name = input_file_list[1] - - # Set output file name - if config.has_option(module_config_sec, "OUTPUT_FILE_PATTERN"): - output_file_pattern = config.get( - module_config_sec, "OUTPUT_FILE_PATTERN" - ) - else: - output_file_pattern = "random_cat" - - # Get number of random objects requested on output - n_rand = config.getfloat(module_config_sec, "N_RANDOM") - - # Flag whether n_rand is total (DENSITY=False, default) - # or per square degree (DENSITY=True) - if config.has_option(module_config_sec, "DENSITY"): - density = config.getboolean(module_config_sec, "DENSITY") - else: - density = False - - # Get healpix output options - save_mask_as_healpix = config.getboolean( - module_config_sec, "SAVE_MASK_AS_HEALPIX" - ) - if save_mask_as_healpix: - healpix_options = {} - for option_trunc in ['FILE_BASE', 'OUT_NSIDE']: - option = f'HEALPIX_OUT_{option_trunc}' - healpix_options[option_trunc] = config.get( - module_config_sec, option - ) - # Create rand cat class instance - rand_cat_inst = RandomCat( - input_image_name, - input_mask_name, - run_dirs["output"], - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ) - - # Run processing - rand_cat_inst.process() - - # No return objects - return None, None diff --git a/uv.lock b/uv.lock index 6a119eea6..e1d016ab6 100644 --- a/uv.lock +++ b/uv.lock @@ -141,24 +141,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/ef/a5cf36a402c4511a776405f12d0b35196f55f4312ce407012a2fbcf1e4e0/astropy-8.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b482bc6c57c966e6c6234410a1d9afbcf92bcac858cf287812f8c99ddc3fafc", size = 10407732, upload-time = "2026-07-05T07:24:40.774Z" }, ] -[[package]] -name = "astropy-healpix" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/c1/aeb3fe3be2ee863708d625014267c71abfe20ddaa293b3d4ddb72ee1d6e9/astropy_healpix-2.0.1.tar.gz", hash = "sha256:0e3f1c94064c45da779900cb90c938df7aef99a924abb23eeb893b16540e77e6", size = 112256, upload-time = "2026-07-20T21:07:30.004Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/3d/0cde0db89ac8dd4e5347322470530298915739f7e9b356b01c1939de8c4f/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:179119d5a69e7b9245919cbe04c3e6bf0a485516b36c29f3402951aad5452251", size = 191178, upload-time = "2026-07-20T21:07:16.512Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c4/71cf2bd4374cc17e015413462be8051fe08bc49e077b7d48072fff4e465d/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c092c54124c48f8d98e04fb22f4b2aa4c8675e65d81c351523f41377f9a6df22", size = 193429, upload-time = "2026-07-20T21:07:18.015Z" }, - { url = "https://files.pythonhosted.org/packages/fa/5c/3a50f68225b836b395da4fb8dfd3d702ded1916042490b16a613caa0e4a6/astropy_healpix-2.0.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce875a29c598c1a99f8f68351daeb4173463044dce4f1c7ebfe8c233ec5e9a49", size = 188694, upload-time = "2026-07-20T21:07:19.244Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a7/c369703ca3fc6f5bee31b3d1f6d0f0b38c15ec84e7fbb6f552c0c7cedfb2/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78f76785852bcc748f5efab8bb4f2ab1fe959d7a998b48e7ed1e59a46cbf0e51", size = 198717, upload-time = "2026-07-20T21:07:24.657Z" }, - { url = "https://files.pythonhosted.org/packages/32/d5/0c4183611b8f36877112882e25cc8a91655a2d11e120ea1f5c153cb7d3a0/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02cefde735da1fe74654e786e02286261b04cc43f5c908a86a8457b2989ac2aa", size = 201205, upload-time = "2026-07-20T21:07:26.128Z" }, - { url = "https://files.pythonhosted.org/packages/e2/54/42bcd02b7c604d3a1132dfa93195bdc2677bb7844172cd098e149cbcb4e9/astropy_healpix-2.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65bee7b70f35ddb81d6b6c849c5bf46768afbce6869395e4f9e0a5c27cb0ce17", size = 196134, upload-time = "2026-07-20T21:07:27.55Z" }, -] - [[package]] name = "astropy-iers-data" version = "0.2026.8.10.0.32.39" @@ -476,15 +458,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -712,45 +685,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] -[[package]] -name = "dask" -version = "2026.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "cloudpickle" }, - { name = "fsspec" }, - { name = "packaging" }, - { name = "partd" }, - { name = "pyyaml" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/39/cbd21c9133d02b4e60899ed466ad5e876553ea68ebffee6209e7dcafc8d4/dask-2026.7.1.tar.gz", hash = "sha256:5727484427665f051e86bf87d021a64d6411141cdc8a20bfe3c1ad2968cc06b7", size = 11548794, upload-time = "2026-07-14T01:06:22.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/5f/7c22733da92b3a6cc4dddcaa8731089d213c2790bbc997e51c429a4e8f8b/dask-2026.7.1-py3-none-any.whl", hash = "sha256:985ffd6c5e9d7979ede515e84ae8d39b647d6aa64f77600f15714ff65f578fe6", size = 1496882, upload-time = "2026-07-14T01:06:20.341Z" }, -] - -[package.optional-dependencies] -array = [ - { name = "numpy" }, -] - -[[package]] -name = "dask-image" -version = "2026.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dask", extra = ["array"] }, - { name = "numpy" }, - { name = "pims" }, - { name = "scipy" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/5b/15d6d6ff8697b188787609be059fe4f07f99fc00f43f68e9e1540fa8733e/dask_image-2026.5.0-py3-none-any.whl", hash = "sha256:acf86cd7f0f1e97804198d30b7cc931efd29f9dec86c65ec004b50405c3f5227", size = 43814, upload-time = "2026-05-27T14:05:56.237Z" }, -] - [[package]] name = "datetime" version = "6.0" @@ -803,18 +737,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - [[package]] name = "dpath" version = "2.2.0" @@ -895,15 +817,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, ] -[[package]] -name = "fsspec" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, -] - [[package]] name = "future" version = "1.0.0" @@ -960,20 +873,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/ed/ae57eb7d344f43f87b74b3a281ead6ec7d6394eef72a7b1dcb28dd089550/gitpython-3.1.59-py3-none-any.whl", hash = "sha256:67a82f537384578643624c8b2c531938a9b82be431663e575dcf638526631d4c", size = 220996, upload-time = "2026-08-10T12:03:18.804Z" }, ] -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, -] - [[package]] name = "greenlet" version = "3.5.5" @@ -1809,15 +1708,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/f9/7b7b50f80b4585bcd78675ff3110c256877b11df32a8cde284f851762f57/llvmlite-0.49.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32adb84fdaae28aeb86fdb6253084ee707ee157289a2e98fe3caf48a62bee82", size = 58344482, upload-time = "2026-08-11T16:25:51.527Z" }, ] -[[package]] -name = "locket" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, -] - [[package]] name = "lsstdesc-coord" version = "1.3.1" @@ -2262,24 +2152,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/f9/3a7b6dbf81e01a48958b45ad2239edbc64707522ab17f11f9f18c44bf6d1/numba-0.67.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83ab968b0e0fa744eba03351282dd8000796e6ec8e4518f47bd3ed86c0a20c7b", size = 3614644, upload-time = "2026-08-11T23:03:55.794Z" }, ] -[[package]] -name = "numcodecs" -version = "0.16.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, - { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, - { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, - { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, - { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, -] - [[package]] name = "numpy" version = "2.5.2" @@ -2380,19 +2252,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] -[[package]] -name = "partd" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "locket" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, -] - [[package]] name = "pexpect" version = "4.9.0" @@ -2446,19 +2305,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, ] -[[package]] -name = "pims" -version = "0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imageio" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "slicerator" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } - [[package]] name = "platformdirs" version = "4.11.3" @@ -2553,15 +2399,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "pyavm" -version = "0.9.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/ac/a925d36dd37fc37f89afccc1f62ffa03b7344c8e4ff7850be7879b4497e6/pyavm-0.9.9.tar.gz", hash = "sha256:bc0f605d957c1fd6d7765523fcba8b9a72377ac6c51461c2a838fe44600bcd9a", size = 220572, upload-time = "2026-03-12T09:54:50.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/ab/ba8d2b40aee05cd986807d58ee324369eb16248b717a08b51eee977f8d33/pyavm-0.9.9-py3-none-any.whl", hash = "sha256:8bba0ee9645a8a9f215af9ceea67b494a6ee1fe380cebdc00ba99d781539ad76", size = 379786, upload-time = "2026-03-12T09:54:49.455Z" }, -] - [[package]] name = "pybind11" version = "3.1.0" @@ -3048,28 +2885,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] -[[package]] -name = "reproject" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "astropy-healpix" }, - { name = "dask", extra = ["array"] }, - { name = "dask-image" }, - { name = "fsspec" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pyavm" }, - { name = "scipy" }, - { name = "zarr" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/5b/8d9b51c754ab014194d374cb4873d0729b397f67997e67721538b643e682/reproject-0.21.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2132fc5d2fa3fbbd57337099f9d47582136e653ac82402783c7ceffa26804816", size = 1776590, upload-time = "2026-06-25T15:11:30.358Z" }, - { url = "https://files.pythonhosted.org/packages/da/2d/f9d76e8e308813978227e1e43a1f25f64cd0f4b030cecaeff8baeec9eeac/reproject-0.21.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8525baaca84e949a69532c02ff491c23993e3bc22a59f694af576e473f2a9d2f", size = 1791889, upload-time = "2026-06-25T15:11:31.741Z" }, -] - [[package]] name = "requests" version = "2.34.2" @@ -3417,7 +3232,6 @@ dependencies = [ { name = "pyqtgraph" }, { name = "python-dateutil" }, { name = "python-pysap" }, - { name = "reproject" }, { name = "sf-tools" }, { name = "skaha" }, { name = "sqlitedict" }, @@ -3503,7 +3317,6 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0" }, { name = "python-dateutil" }, { name = "python-pysap", specifier = ">=0.3" }, - { name = "reproject", specifier = ">=0.19" }, { name = "ruff", marker = "extra == 'lint'" }, { name = "sf-tools", specifier = ">=2.0.4" }, { name = "shapepipe", extras = ["doc", "jupyter", "lint", "release", "test", "fitsio"], marker = "extra == 'dev'" }, @@ -3556,15 +3369,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/d3/a3c1d569ae714ed9985adccefdfe2d6dd9f8ad7a297bd8fa4c7ed30c587b/skaha-1.7.0-py3-none-any.whl", hash = "sha256:68b0d3c925b98bf145c5f695237474fe1cea07cf9b69b4a9bae9910375dfa01a", size = 40795, upload-time = "2025-05-28T21:17:02.296Z" }, ] -[[package]] -name = "slicerator" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/52/f38586b82b2935f8b59a09b0a79c545a22ed062e728c9418bafeb51f61e0/slicerator-1.1.0.tar.gz", hash = "sha256:44010a7f5cd87680c07213b5cabe81d1fb71252962943e5373ee7d14605d6046", size = 38283, upload-time = "2022-04-07T18:54:08.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, -] - [[package]] name = "smart-open" version = "7.7.1" @@ -4027,15 +3831,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, ] -[[package]] -name = "toolz" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, -] - [[package]] name = "tornado" version = "6.5.8" @@ -4251,23 +4046,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, ] -[[package]] -name = "zarr" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "donfig" }, - { name = "google-crc32c" }, - { name = "numcodecs" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/15/436cb1d3bbe86173bd44ce7a34ecb210d0c0416946e337858149a905ef5a/zarr-3.3.0.tar.gz", hash = "sha256:cd0c8cf738b4bb4807815bc1255acad5bdf1a7b7264b606c5a1bc0d0392a306b", size = 943626, upload-time = "2026-07-30T16:35:10.491Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/c6/6b726ddf4c3ac5a123f285c3650fa8268902c28ea541a0282867f1336e65/zarr-3.3.0-py3-none-any.whl", hash = "sha256:323bf5366d4f909052ef6e2e03e7481a7434c3ee75d3a981eb3a71fc1ae22cef", size = 363685, upload-time = "2026-07-30T16:35:08.794Z" }, -] - [[package]] name = "zipp" version = "4.1.0" From edec5460b449a57906df566a6ac293c35c0562a5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 11:36:38 -0400 Subject: [PATCH 16/17] =?UTF-8?q?fix(config):=20the=202026=20DR6=20product?= =?UTF-8?q?s=20are=20named=20mask=5Fugriz=5F*=20=E2=80=94=20point=20the=20?= =?UTF-8?q?placeholder=20at=20the=20real=20ladder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Aug-2026 post-GSC2 band-combined healsparse products (arc:home/mhudson/masks/) use the mask_ugriz_nside131072_n.hsp naming; the mask_r_* files on canfar's ShapePipe/mask are the 2025 vintage. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- example/cfis/config_exp_mccd.ini | 2 +- example/cfis/config_exp_psfex.ini | 2 +- src/shapepipe/utilities/mask_query.py | 2 +- workflow/config/cfis/config_exp_psfex.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 5d2460312..68dcfbeef 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -145,7 +145,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index b4e2d1fa1..2f1c8e353 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -147,7 +147,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index d789d4100..e5c6c02eb 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -33,7 +33,7 @@ edge convention we did not think of. ``test_partial_read_matches_full`` is what actually holds the two paths equal. -Measured on the DR6 star map (``mask_r_nside131072_n4.hsp``, 583 MB, +Measured on the DR6 star map (``mask_ugriz_nside131072_n4.hsp``, 583 MB, ``nside_coverage=128``) for 2000 positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB peak RSS, full 12.8 s / 3364 MiB, identical values. Per 40-CCD exposure that is ~4 s against ~8.5 min of map reading, and diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 2fa5dc525..ffcac86b6 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -146,7 +146,7 @@ NUMBERING_SCHEME = -0000000-0 # adding paths here; every map that is True (boolean) or nonzero (integer) at a # detection sets FLAG_EXT, which star_selection.setools cuts on as # FLAG_EXT == 0. Comma-separated. -MASK_PATHS = $SP_CONFIG/mask_r_nside131072_n4.hsp +MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, # any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map From 97c44b759f4ce00fb6339e7ea9ebedb8bfc30d9a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 12:21:36 -0400 Subject: [PATCH 17/17] =?UTF-8?q?feat(mask=5Fquery):=20ship=20permissive?= =?UTF-8?q?=20=E2=80=94=20flag=20PSF-star=20candidates,=20cut=20nothing=20?= =?UTF-8?q?by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PSF star selection no longer cuts on the external masks. mask_query stays in the chain and keeps writing FLAG_EXT from the shipped MASK_PATHS (the star-body map, bit 2); star_selection.setools drops `FLAG_EXT == 0` from all four mask blocks and rejects on the instrument flags alone, leaning on outlier rejection for the rest. This makes the exposure side agree with the tile side, which was already permissive: make_cat writes MASK_ unfiltered. Flag transparently, cut downstream — so the effect of a mask on the star sample can be MEASURED before it is imposed, rather than baked in on the way past. The escape hatch is documented where someone would look for it, because a permissive default is only safe if reversing it is obvious. star_selection.setools' header now spells out the change (add `FLAG_EXT == 0` beside each `IMAFLAGS_ISO == 0`, one line per block) and says why it exists: if outlier rejection turns out not to be robust enough. The module docstring gains a "Nothing cuts on it by default" section, and the config comments beside MASK_PATHS say NOTHING CUTS ON IT rather than naming a cut that is no longer there. The tutorial's Masks section, pipeline_canfar.md and workflow/README.md lose the same stale claim; the shared utility now says setools *could* cut on the column, not that it does. The diet section is retitled: MASK_PATHS is a list of maps to RECORD against each candidate, not to reject on. Halos stay out for the same reason as before — they say nothing about whether a star is a good PSF sample. Verified: 25 tests pass, 29 runners import, 64 configs resolve, ruff clean. Only the four cut lines changed in the setools body — the mask blocks and their IMAFLAGS_ISO tests are otherwise untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Cem4A9vjxA7nkPnyBKrc5W --- docs/source/pipeline_canfar.md | 2 +- docs/source/pipeline_tutorial.md | 16 ++++++-- example/cfis/config_exp_mccd.ini | 6 ++- example/cfis/config_exp_psfex.ini | 6 ++- example/cfis/star_selection.setools | 32 ++++++++------- .../modules/mask_query_package/__init__.py | 39 +++++++++++++------ src/shapepipe/utilities/mask_query.py | 3 +- workflow/README.md | 8 ++-- workflow/config/cfis/config_exp_psfex.ini | 6 ++- 9 files changed, 77 insertions(+), 41 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 926d36578..1555f75f2 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -174,7 +174,7 @@ shapepipe_run -c cfis/config_tile_Uz.ini There is no masking step. `ShapePipe` generates no masks: the sky-fixed healsparse maps are queried once per object, by `mask_query` on the exposure -catalogues (`FLAG_EXT`, cut by `setools`) and by `make_cat` on the final tile +catalogues (`FLAG_EXT`, recorded but not cut on) and by `make_cat` on the tile catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` config entries at the maps and nothing else is needed — no star-catalogue download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index a4db96579..b2cf0640f 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -196,15 +196,23 @@ consumed by *querying them at object positions*, never by rasterizing them onto pixels. Two modules do the querying, from the same shared lookup (`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and `setools` on the single-exposure single-CCD catalogues and writes one integer -`FLAG_EXT` column (0 = clean), which `star_selection.setools` cuts on so that -masked objects never enter the PSF star sample — a deliberately narrow diet, -the star-body map (bit 2) only, since halos flag objects without disqualifying -them as PSF stars; `make_cat` writes one +`FLAG_EXT` column (0 = clean), recording the star-body map (bit 2) against every +detection; `make_cat` writes one `MASK_` column per band onto the final tile catalogue, carrying the map value verbatim so downstream selections choose their own cuts. Map paths and bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and `MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. +**Nothing in the pipeline cuts on these columns.** The PSF star selection ships +permissive: `star_selection.setools` rejects on the instrument flags +(`IMAFLAGS_ISO == 0`) and nothing else, leaning on outlier rejection for the +rest, and the final catalogue's `MASK_` columns are written unfiltered. +The principle is to flag transparently and cut downstream, so the effect of a +mask can be measured before it is imposed. If outlier rejection turns out not to +be robust enough, feeding the external masks to the star selection is one line +per mask block — add `FLAG_EXT == 0` beside each `IMAFLAGS_ISO == 0` — and that +file's header documents the change. + No internet access is needed at any point, and there is no reference star catalogue to download. diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini index 68dcfbeef..ec43bbf46 100644 --- a/example/cfis/config_exp_mccd.ini +++ b/example/cfis/config_exp_mccd.ini @@ -143,8 +143,10 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT, which star_selection.setools cuts on as -# FLAG_EXT == 0. Comma-separated. +# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships +# permissive (instrument flags only) and the column is carried for +# transparency and measurement — see star_selection.setools' header for the +# one-line change that would impose it. Comma-separated. MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini index 2f1c8e353..afa41cc7c 100644 --- a/example/cfis/config_exp_psfex.ini +++ b/example/cfis/config_exp_psfex.ini @@ -145,8 +145,10 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT, which star_selection.setools cuts on as -# FLAG_EXT == 0. Comma-separated. +# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships +# permissive (instrument flags only) and the column is carried for +# transparency and measurement — see star_selection.setools' header for the +# one-line change that would impose it. Comma-separated. MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index 230017053..bbda2e520 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -1,16 +1,24 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties ## -## Two independent mask cuts, and they come from different places: +## ONE mask cut, and it is the instrument flags: ## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation), -## delivered with the exposure and read by SExtractor; -## FLAG_EXT == 0 the external healsparse masks, queried per detection by -## the mask_query module. Which maps reach FLAG_EXT is that -## module's MASK_PATHS config, and the shipped diet is -## deliberately narrow: the star-body map (bit 2) only, no -## halos (they flag, they do not reject stars) and no -## MaxiMask. -## SETools expressions have no bitwise operators, so mask_query does the bit -## selection and this file only tests for zero. +## delivered with the exposure and read by SExtractor. +## +## The external healsparse masks are NOT cut on here. mask_query queries them +## per detection and writes FLAG_EXT into this catalogue, but the selection +## ships permissive: the column is carried for transparency and measurement, +## and the star sample is defended by outlier rejection instead. Flag +## transparently, cut downstream. +## +## To feed the external masks to the star selection after all, add +## +## FLAG_EXT == 0 +## +## beside each IMAFLAGS_ISO line below. That is the whole change — one line per +## mask block — and it is the intended escape hatch if outlier rejection turns +## out not to be robust enough. Which maps reach FLAG_EXT is mask_query's +## MASK_PATHS config; SETools has no bitwise operators, so the bit selection +## happens there and this file would only ever test for zero. [MASK:preselect] MAG_AUTO > 0 @@ -19,13 +27,11 @@ FWHM_IMAGE > 0.3 / 0.187 FWHM_IMAGE < 1.5 / 0.187 FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 NO_SAVE [MASK:flag] FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 NO_SAVE @@ -37,7 +43,6 @@ FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 [MASK:fwhm_mag_cut] FWHM_IMAGE > 0 @@ -45,7 +50,6 @@ FWHM_IMAGE < 40 MAG_AUTO < 35 FLAGS == 0 IMAFLAGS_ISO == 0 -FLAG_EXT == 0 NO_SAVE # Split the 'star_selection' sample into diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py index 0391e5393..3acb8820e 100644 --- a/src/shapepipe/modules/mask_query_package/__init__.py +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -29,24 +29,39 @@ carry its bits through. Nothing downstream reads more than ``== 0``. The single column exists because ``setools`` expressions support only -``< > <= >= == !=`` — no bitwise operators — so the bit selection has to -happen here. ``star_selection.setools`` cuts on ``FLAG_EXT == 0`` beside its -existing ``IMAFLAGS_ISO == 0``: instrument flags reject pixels, the queried -masks reject objects. +``< > <= >= == !=`` — no bitwise operators — so any bit selection has to happen +here, leaving the config a plain test for zero. + +Nothing cuts on it by default +============================= + +The shipped ``star_selection.setools`` does NOT cut on ``FLAG_EXT``. The PSF +star selection is permissive: it rejects on the instrument flags +(``IMAFLAGS_ISO == 0``) and leans on outlier rejection for the rest. The +column is written for transparency and measurement — so the effect of the +external masks on the star sample can be *measured* before it is imposed — +which is the same principle as ``make_cat``'s unfiltered ``MASK_`` +columns: flag here, cut downstream. + +Feeding the external masks to the star selection is a one-line change if +outlier rejection turns out not to be robust enough: add ``FLAG_EXT == 0`` by +each +``IMAFLAGS_ISO == 0`` in ``star_selection.setools``. The escape hatch is +deliberate, and the file's header says so. The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with ``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is written once. That module's docstring documents the off-coverage convention. -The diet is deliberately narrow -=============================== +What gets queried is deliberately narrow +======================================== -``MASK_PATHS`` is a *list of maps to reject PSF stars on*, not a list of every -mask that exists. The committed configs name exactly one map — the UNIONS -star-body product (bit 2) — beside the instrument flags SExtractor already -reads. Halo bits 0 and 1 are excluded on purpose: halos flag objects for the -final catalogue, they do not reject PSF stars (mask-force telecon, 2026-07-21). -MaxiMask is not in the diet either. +``MASK_PATHS`` is a *list of maps to record against each PSF-star candidate*, +not a list of every mask that exists. The committed configs name exactly one +map — the UNIONS star-body product (bit 2). Halo bits 0 and 1 are excluded on +purpose: halos flag objects for the final catalogue, they say nothing about +whether a star is a good PSF sample (mask-force telecon, 2026-07-21). MaxiMask +is not queried here either. Widening it costs a config edit and no code — add a path. That is why the contract is a path list rather than a bit mask: the UNIONS products are one diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py index e5c6c02eb..cf79b21e0 100644 --- a/src/shapepipe/utilities/mask_query.py +++ b/src/shapepipe/utilities/mask_query.py @@ -15,7 +15,8 @@ * ``mask_query`` writes a single integer ``FLAG_EXT`` column onto the exposure SExtractor catalogue, combining the configured maps into "clean (0) or flagged (nonzero)" so that ``setools`` — whose expression language has no - bitwise operators — can cut on ``FLAG_EXT == 0``. + bitwise operators — *could* cut on ``FLAG_EXT == 0``. The shipped selection + does not; the column is carried for measurement (see that module). Partial reads ------------- diff --git a/workflow/README.md b/workflow/README.md index ac997afef..4aa24d29b 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -201,9 +201,11 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed healsparse maps and QUERIED once per object: the `mask_query` module writes a - `FLAG_EXT` column onto each CCD's detection catalogue for setools' star cut - (inside `exp_psf`), and `make_cat` writes one `MASK_` column per band - onto the final catalogue (inside `tile_make_cat`). Map paths are config, not + `FLAG_EXT` column onto each CCD's detection catalogue (inside `exp_psf`), and + `make_cat` writes one `MASK_` column per band onto the final catalogue + (inside `tile_make_cat`). Neither is cut on in the pipeline: the PSF star + selection rejects on instrument flags alone and everything else is a + downstream decision. Map paths are config, not code, so regenerated products cost a config edit. Nothing is fetched from a catalogue server, staged, or rasterized, which is why the old `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index ffcac86b6..858e3828f 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -144,8 +144,10 @@ NUMBERING_SCHEME = -0000000-0 # objects for the final catalogue, they do not reject PSF stars (mask-force # telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by # adding paths here; every map that is True (boolean) or nonzero (integer) at a -# detection sets FLAG_EXT, which star_selection.setools cuts on as -# FLAG_EXT == 0. Comma-separated. +# detection sets FLAG_EXT. NOTHING CUTS ON IT: the star selection ships +# permissive (instrument flags only) and the column is carried for +# transparency and measurement — see star_selection.setools' header for the +# one-line change that would impose it. Comma-separated. MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp # Optional: restrict integer maps to these bits (value & MASK_BITS). Absent,