diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 53e67dcca..6de369a66 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -83,12 +83,11 @@ jobs: # Smoke-test the binaries baked into the runtime image. Catches the # class of regression where the image builds but a runtime tool - # (sextractor, weightwatcher) is missing or unrunnable. + # (sextractor, psfex) is missing or unrunnable. - name: Test runtime — binaries run: | IMAGE=$(echo "${{ steps.meta-runtime.outputs.tags }}" | head -n1) docker run --rm "$IMAGE" source-extractor --version - docker run --rm "$IMAGE" weightwatcher --version docker run --rm "$IMAGE" psfex --version - name: Test runtime — shapepipe entry point (read-only fs) diff --git a/Dockerfile b/Dockerfile index 0a549ae8f..f6d323706 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV SHELL=/bin/bash \ COVERAGE_FILE=/tmp/.coverage # System dependencies — three categories: -# - astromatic binaries (psfex, source-extractor, weightwatcher) ship as +# - astromatic binaries (psfex, source-extractor) ship as # Debian packages on bookworm; preferred over building from source. # - compilers and dev libs needed to build the heavier wheels (galsim, # mpi4py, python-pysap, fitsio). @@ -55,7 +55,7 @@ RUN apt-get update -y --quiet && \ libcfitsio-dev \ libproj-dev proj-bin \ libgl1-mesa-glx \ - psfex source-extractor weightwatcher && \ + psfex source-extractor && \ apt-get clean && rm -rf /var/lib/apt/lists/* # OpenMPI from source — required for hybrid Apptainer MPI on HPC clusters. diff --git a/docs/source/container.md b/docs/source/container.md index 174cde9dc..1a83b0f55 100644 --- a/docs/source/container.md +++ b/docs/source/container.md @@ -165,7 +165,7 @@ The Dockerfile does **not** duplicate Python deps — those come from The asymmetry is deliberate: Python deps go through pyproject + lockfile (reproducible, auditable), system deps go through Dockerfile (Debian's versioning). Don't `apt install` something that has a Python wheel; don't -`pip install` something Debian packages directly (e.g. `weightwatcher`). +`pip install` something Debian packages directly (e.g. `source-extractor`). ## Why this shape @@ -175,8 +175,8 @@ versioning). Don't `apt install` something that has a Python wheel; don't - **`uv sync --frozen`** at build time means the image is bit-exactly reproducible from a tagged commit, and impossible to ship with a stale lockfile. -- **Astromatic binaries from Debian** (`psfex`, `source-extractor`, - `weightwatcher`) instead of source builds — Debian carries the +- **Astromatic binaries from Debian** (`psfex`, `source-extractor`) + instead of source builds — Debian carries the GCC-compatibility patches that the previous Dockerfile had to apply inline with `sed`. - **Two targets** so canfar batch deployments stay slim while interactive diff --git a/docs/source/dependencies.md b/docs/source/dependencies.md index 9378bfe00..b907a452d 100644 --- a/docs/source/dependencies.md +++ b/docs/source/dependencies.md @@ -57,7 +57,6 @@ packages (no source builds), plus the MPI stack: |---------|------------| | [Source Extractor](https://www.astromatic.net/software/sextractor/) | {cite:p}`bertin:96` | | [PSFEx](https://www.astromatic.net/software/psfex/) | {cite:p}`bertin:11` | -| [WeightWatcher](https://www.astromatic.net/software/weightwatcher/) | {cite:p}`marmo:08` | | OpenMPI (5.0.x) | | Python dependencies themselves are managed with [uv](https://docs.astral.sh/uv/); diff --git a/docs/source/installation.md b/docs/source/installation.md index 83df0b6bb..ee3f8e182 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -38,8 +38,8 @@ docker pull ghcr.io/cosmostat/shapepipe:develop-runtime We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance. ``` -The image bundles the astromatic binaries (`source-extractor`, `psfex`, -`weightwatcher`), MPI (`mpi4py` + OpenMPI), and every Python dependency, so +The image bundles the astromatic binaries (`source-extractor`, `psfex`), +MPI (`mpi4py` + OpenMPI), and every Python dependency, so there is nothing else to install or build. To process data on a cluster with MPI, run the pipeline through Apptainer the same way you would any MPI job. diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 96add5758..ecc621e9f 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -170,22 +170,16 @@ The downloaded tile weights are compressed. The following call uncompresses all. shapepipe_run -c cfis/config_tile_Uz.ini ``` -### Mask tiles +### Masks -This step is done globally for all tiles. There might be job failures or interruptions. The following -command to the `ShapePipe` job script can be run repeatedly; already created masks will be skipped. - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 4 -``` - -If masks were created in more than one run, i.e. situated in more than one output directory, these have to be -combined for subsequent pipeline module runs. This is done by creating a new output directory with symbolic -links, using the script - -```bash -combine_runs.bash -c flag_tile -``` +There is no masking step. `ShapePipe` generates no masks: the sky-fixed +healsparse maps are queried once per object, by `mask_query` on the exposure +catalogues (`MASK_EXT`, recorded but not cut on) and by `make_cat` on the tile +catalogue (`MASK_` columns). Point the `MASK_PATHS` / `MASK_EXT_PATHS` +config entries at the maps and nothing else is needed — no star-catalogue +download, no rasterization, no `combine_runs.bash -c flag_*`. The only mask that +touches pixels is the instrument flag image shipped with each exposure, which +`split_exp` splits per CCD. ## Tile detection @@ -205,7 +199,7 @@ canfar_submit_job -j 16 -f tile_numbers.txt -P N_PAR -v -J JMAX ### Exposure Processing -#### Option 0: Global split and exp masks (deprecated; used for earlier v1.x patch runs) +#### Option 0: Global split (deprecated; used for earlier v1.x patch runs) For this option, set `sp_local=0`. @@ -217,21 +211,7 @@ For `sp_local=-` both `mh_local` (0, 1) are ok: export mh_local=0 ``` -#### Option 0: Mask exposures (deprecated) - -Run repeatedly if necessary: - -```bash -job_sp_canfar.bash -p $psf -n $OMP_NUM_THREADS -j 8 -``` - -Combine all runs: - -```bash -combine_runs.bash -c flag_exp -``` - -### Option 1: Local split and mask exposures (recommended) +### Option 1: Local split exposures (recommended) Optional: Enable flags for local split processing and merge header runs as @@ -258,12 +238,6 @@ First, determine the number of maximum jobs with the option `-s` (see above). Th canfar_submit_job -j 2 -v -f exp_shdu.txt -v -P N_PAR -J JMAX ``` -### Mask exposures - -```bash -canfar_submit_job -j 8 -f exp_shdu.txt -v -P N_PAR -J JMAX -``` - ### Exposure detection ```bash diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index 95a0aa94a..51f09da7a 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -44,11 +44,11 @@ Naming and numbering of the input files can closely follow the original image na A stacked image is also called *tile*. These files are used on input by `ShapePipe`. The pixel data can contain the observed image, a weight map, or a flag map. Tile images and weights are created in the case of CFIS by Stephen Gwyn using a combination of `swarp` and his own software. Examples of file names are - `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tile flag files - are created the mask module of `ShapePipe` (see [Mask images](#mask-images)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for + `CFIS.316.246.r.fits`, `CFIS.205.267.r.weight.fits.fz`, the latter is a compressed FITS file, see below. Tiles have no flag file + (see [Masks](#masks)). The tile ID needs to be modified such that the `.` between the two tile numbers (RA and DEC indicator) is not mistaken for a file extension delimiter. For the same reason, the extension `.fits.fz` is changed to `.fitzfz`. In addition, for clarity, we include the string `image` for a tile image type. Default convention: **-.fits** - Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz`, `pipeline_flag-239-293.fits` + Examples: `CFIS_image-277-282.fits`, `CFIS_weight-274-282.fitsfz` - Database catalogue files For very large files that combine information from multiple tiles or single exposures, `ShapePipe` creates `sqlite` @@ -128,8 +128,6 @@ for all options. This script creates the subdirectory `$SP_RUN/output` to store all pipeline outputs (log files, diagnostics, statistics, output images, catalogues, single-exposure headers with WCS information). -Optionally, the subdir `output_star_cat` is created by the used to store the external star catalogues for masking. This is only necessary if the pipeline is run on a cluster without internet connection to access star catalogues. In that case, the star catalogues need to be retrieved outside the pipeline, for example on a login node, and copied to `output_star_cat`. - The job script automaticall performs a number of subsequent calls to the `ShapePipe` executable `shapepipe_run`, as ```bash shapepipe_run -c $SP_CONFIG/.ini @@ -189,32 +187,49 @@ Finally, the headers of all single-exposure single-CCD files are merged into a s Two output directories are created, `run_sp_Uz` for `uncompress_fits`, and `run_sp_exp_SpMh` for the output of the modules `split_exp` (`Sp`) and `merge_headers` (`Mh`). -## Mask images - -Run -```bash -job_sp TILE_ID -j 4 -``` -to mask tile and single-exposure single-CCD images. Both tasks are performed by two calls to the `mask` runner. - -Note that internet access is required for this step, since a reference star catalogue is downloaded. - -The output of both masking runs are stored in the output directory `run_sp_MaMa`, with run 1 (2) of -`mask` corresponding to tiles (exposures). - -**Diagnostics:** Open a single-exposure single-CCD image and the corresponding pipeline flag -in `ds9`, and display both frames next to each other. Example -```bash -ds9 image-2113737-10.fits pipeline_flag-2113737-10.fits -``` -Choose `zoom fit` for both frames, click `scale zscale` for the image, and `color aips0` for the flag, to display something like this: - - - -By eye the correspondence between the different flag types and the image can be -seen. Note that the two frames might not match perfectly, since (a) WCS -information is not available in the flag file FITS headers; (b) the image can -have a zero-padded pixel border, which is not accounted for by `ds9`. +## Masks + +`ShapePipe` does not generate masks. Sky-fixed masks — star halos, stars, +manual masks for large galaxies, per-band coverage, MaxiMask defects — are +supplied as [healsparse](https://healsparse.readthedocs.io) maps and are +consumed by *querying them at object positions*, never by rasterizing them onto +pixels. Two modules do the querying, from the same shared lookup +(`shapepipe.utilities.mask_query`): `mask_query` runs between `sextractor` and +`setools` on the single-exposure single-CCD catalogues and writes one integer +`MASK_EXT` column (0 = clean), recording the star-body map (bit 2) against every +detection; `make_cat` writes one +`MASK_` column per band onto the final tile catalogue, carrying the map +value verbatim so downstream selections choose their own cuts. Map paths and +bit selections live in the config files (`MASK_PATHS` / `MASK_BITS` and +`MASK_EXT_PATHS`), so regenerated mask products cost a config edit and no code. + +The distinction that drives all of this is what a mask *means*. An **instrument +flag** marks a corrupted measurement — the pixels carry no usable signal — so +these are the only masks that reject anything inside the pipeline. The +**healsparse masks** are sky-fixed location flags: they say where an object +sits, not that its pixels are broken, so what to do about one is an analysis +decision and is made downstream. + +**Nothing in the pipeline cuts on the queried columns**, and on exposures the +query ships off entirely: `MASK_PATHS` is commented out, which makes +`mask_query` a strict no-op that passes the catalogue through with no +`MASK_EXT` column (the module stays in the chain, so enabling it is +uncommenting one line). `star_selection.setools` rejects on `IMAFLAGS_ISO == 0` +and nothing else, deliberately starting from outlier rejection alone, and the +final catalogue's `MASK_` columns are written unfiltered. `MASK_EXT` is +the configurable pickup if outlier rejection proves insufficient: add +`MASK_EXT == 0` beside each `IMAFLAGS_ISO == 0`, one line per mask block, as +that file's header documents. + +No internet access is needed at any point, and there is no reference star +catalogue to download. + +The one mask that still reaches pixels is the **instrument flag image** +(`p.flag.fits.fz`) delivered with each exposure, which records bad columns +and saturation. `split_exp` splits it per CCD beside the image and weight, +`sextractor` reads it as `IMAFLAGS_ISO`, and `ngmix` zero-weights flagged +pixels in its postage stamps. Tiles have no such image, so tile detection runs +with `FLAG_IMAGE = False`. ## Detect objects on tiles and process stars on single exposures diff --git a/docs/source/random_cat.md b/docs/source/random_cat.md deleted file mode 100644 index a930d40ed..000000000 --- a/docs/source/random_cat.md +++ /dev/null @@ -1,128 +0,0 @@ -# Create random catalogues and masks - -This section describes how to create tile-based random catalogues and healpix -masks, and combined randoms and masks for a selection of tiles. - -The masked regions are obtained on input from ShapePipe pixel mask ("pipeline flag") -files. - -```{note} -Parts of this procedure use the legacy canfar-VM / `vos` retrieval workflow (see -[VOSpace retrieval](vos_retrieve.md)) and the obsolete `prepare_tiles_for_final` -helper, which is no longer shipped. The `random_cat` module itself is current; -the input-staging and joint-mask steps now overlap with -[`sp_validation`](https://github.com/CosmoStat/sp_validation). The steps are -retained for reference. -``` - -## Set up - -### ID file and shell variables - -First, if if does not exist already, create the file ``tile_numbers.txt`` containing a list of tile IDs, -one per line. This is the same format as the input file to ``get_images_runner``. -For example, link to a patch ID list, -```bash -ln -s tiles_PX.txt tile_numbers.txt -``` -Next, set the run and config paths, -```bash -export SP_RUN=. -export SP_CONFIG=/path/to/config-files -``` - -### Get images or image headers - -We need to footprint of the image tiles. If they have been downloaded for a ``ShapePipe`` run, -check that they are accessible as last run of the ``get_images_runner`` module. - -If not, we can just download the headers to gain significant download time. -```bash -shapepipe_run -c $SP_CONFIG/config_get_tiles_vos_headers.ini -``` - -### Check pixel mask files - -Make sure that all pixel mask files are present. If they have been downloaded from ``vos`` as ``.tgz`` files, -type -```bash -canfar_avail_results -i tile_numbers.txt --input_path . -v -m -o missing_mask.txt -``` -In case of missing mask files, check whether they are present in the ``vos`` remote directory, -```bash -canfar_avail_results -i tile_numbers.txt --input_path vos:cfis/vos-path/to/results -v -m -``` -If missing on ``vos``, process those tiles. If processing only up the the mask is necessary, -the following steps can be carried out, -```bash -job_sp -j 7 TILE_ID -job_sp -j 128 TILE_ID -``` -The first command processes the tile up to the mask; the second line uploads the mask files -to ``vos``. - -Now, download the missing masks with -```bash -canfar_download_results -i missing_mask.txt --input_vos vos-path/to/results -m -v -``` -Untar .tgz files if required, -```bash -while read p; do tar xvf pipeline_flag_$p.tgz; done column; no filtering +# is applied here. Absent this key, the step is a no-op. +; MASK_EXT_PATHS = r:$SP_CONFIG/mask_r.hsp, u:$SP_CONFIG/mask_u.hsp diff --git a/example/cfis/config_onthefly.mask b/example/cfis/config_onthefly.mask deleted file mode 100644 index 7c185c602..000000000 --- a/example/cfis/config_onthefly.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_save.mask b/example/cfis/config_save.mask deleted file mode 100644 index 497dedda9..000000000 --- a/example/cfis/config_save.mask +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis/config_tile_Ma_onthefly.ini b/example/cfis/config_tile_Ma_onthefly.ini deleted file mode 100644 index 8f7ef4206..000000000 --- a/example/cfis/config_tile_Ma_onthefly.ini +++ /dev/null @@ -1,82 +0,0 @@ -# ShapePipe configuration file for masking of tiles - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini index 12f158508..80350eead 100644 --- a/example/cfis/config_tile_Sx.ini +++ b/example/cfis/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options @@ -55,11 +60,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner, run_sp_tile_Mh_exp:merge_headers_runner +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag, log_exp_headers +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers -FILE_EXT = .fits, .fits, .fits, .sqlite +FILE_EXT = .fits, .fits, .sqlite # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 @@ -69,14 +74,14 @@ EXEC_PATH = source-extractor # SExtractor configuration files DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param DOT_CONV_FILE = $SP_CONFIG/default.conv # Use input weight image if True WEIGHT_IMAGE = True # Use input flag image if True -FLAG_IMAGE = True +FLAG_IMAGE = False # Use input PSF file if True PSF_FILE = False diff --git a/example/cfis/config_tile_Sx_nomask.ini b/example/cfis/config_tile_Sx_nomask.ini deleted file mode 100644 index 731a8d338..000000000 --- a/example/cfis/config_tile_Sx_nomask.ini +++ /dev/null @@ -1,114 +0,0 @@ -# ShapePipe configuration file for tile detection - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER] - -INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner - -FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers - -FILE_EXT = .fits, .fits, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = False - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis/config_tile_onthefly.mask b/example/cfis/config_tile_onthefly.mask deleted file mode 100644 index 69ad20769..000000000 --- a/example/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/config_tile_save.mask b/example/cfis/config_tile_save.mask deleted file mode 100644 index 82c4b66af..000000000 --- a/example/cfis/config_tile_save.mask +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -#CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 0 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = True - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = True - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = True - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = True - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis/defunct/config_MaMa_onthefly.ini b/example/cfis/defunct/config_MaMa_onthefly.ini deleted file mode 100644 index 84f117e65..000000000 --- a/example/cfis/defunct/config_MaMa_onthefly.ini +++ /dev/null @@ -1,105 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_MaMa_save.ini b/example/cfis/defunct/config_MaMa_save.ini deleted file mode 100644 index 4bd1b00ef..000000000 --- a/example/cfis/defunct/config_MaMa_save.ini +++ /dev/null @@ -1,109 +0,0 @@ -# ShapePipe configuration file for masking of tiles and exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_MaMa - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner, mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 8 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER_RUN_1] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, star_cat_tiles - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_image, CFIS_weight, star_cat - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -### Mask exposures -[MASK_RUNNER_RUN_2] - -# Parent module -INPUT_DIR = last:split_exp_runner, star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .cat - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_save.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini b/example/cfis/defunct/config_tile_Sx_exp_mccd.ini deleted file mode 100644 index fec79f177..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_mccd.ini +++ /dev/null @@ -1,274 +0,0 @@ -# ShapePipe configuration file for single-exposures, MCCD PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -## Detection on tile -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -#CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -## Detection on single exposures -[SEXTRACTOR_RUNNER_RUN_2] - -INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2 - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner_run_2 - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) if True -DETECTION_WEIGHT = False - -# Se to True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[MCCD_PREPROCESSING_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -INPUT_DIR = last:setools_runner - -# Input are individual CCDs, thus single-exposure single-HDU images -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - - -[MCCD_FIT_VAL_RUNNER] - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -NUMBERING_SCHEME = -0000000 - - -[MERGE_STARCAT_RUNNER] - -INPUT_DIR = last:mccd_fit_val_runner - -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini - -MODE = FIT_VALIDATION - -VERBOSE = False - -PSF_MODEL = mccd - -NUMBERING_SCHEME = -0000000 - - -[MCCD_PLOTS_RUNNER] - -# Now MCCD has created a focal-plane PSF model, including all CCDS per images, -# thus single-exposure files -NUMBERING_SCHEME = -0000000 - -PSF = mccd - -PLOT_MEANSHAPES = True - -# X_GRID, Y_GRID: correspond to the number of bins in each direction of each -# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. -X_GRID = 5 -Y_GRID = 10 - -PLOT_HISTOGRAMS = True - -# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape -# before drawing the plots. -REMOVE_OUTLIERS = False - diff --git a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini b/example/cfis/defunct/config_tile_Sx_exp_psfex.ini deleted file mode 100644 index ea86ea048..000000000 --- a/example/cfis/defunct/config_tile_Sx_exp_psfex.ini +++ /dev/null @@ -1,248 +0,0 @@ -# ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx_exp_SxSePsf - -# Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 40 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[SEXTRACTOR_RUNNER_RUN_1] - -INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 - -INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 - -FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag - -FILE_EXT = .fits, .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# SExtractor executable path -EXEC_PATH = source-extractor - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_tile.sex -DOT_PARAM_FILE = $SP_CONFIG/default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -ZP_FROM_HEADER = False - -BKG_FROM_HEADER = False - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, -# MINIBACK_RMS, -BACKGROUND, #FILTERED, -# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -MAKE_POST_PROCESS = True - -# Multi-epoch mode: Path to file with single-exposure WCS header information -LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD - -# Number of pixels in x,y of a CCD. Format: Nx,Ny -CCD_SIZE = 33,2080,1,4612 - - -[SEXTRACTOR_RUNNER_RUN_2] - -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . - -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner - -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag - -NUMBERING_SCHEME = -0000000-0 - -# SExtractor executable path -EXEC_PATH = sex - -# SExtractor configuration files -DOT_SEX_FILE = $SP_CONFIG/default_exp.sex -DOT_PARAM_FILE = $SP_CONFIG//default.param -DOT_CONV_FILE = $SP_CONFIG/default.conv - -# Use input weight image if True -WEIGHT_IMAGE = True - -# Use input flag image if True -FLAG_IMAGE = True - -# Use input PSF file if True -PSF_FILE = False - -# Use distinct image for detection (SExtractor in -# dual-image mode) if True. -DETECTION_IMAGE = False - -# Distinct weight image for detection (SExtractor -# in dual-image mode) -DETECTION_WEIGHT = False - -# True if photometry zero-point is to be read from exposure image header -ZP_FROM_HEADER = True - -# If ZP_FROM_HEADER is True, zero-point key name -ZP_KEY = PHOTZP - -# Background information from image header. -# If BKG_FROM_HEADER is True, background value will be read from header. -# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. -# This is used e.g. for the LSB images. -BKG_FROM_HEADER = False -# LSB images: -# BKG_FROM_HEADER = True - -# If BKG_FROM_HEADER is True, background value key name -# LSB images: -#BKG_KEY = IMMODE - -# Type of image check (optional), default not used, can be a list of -# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, -# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND - -# File name suffix for the output sextractor files (optional) SUFFIX = tile -SUFFIX = sexcat - -## Post-processing - -# Not required for single exposures -MAKE_POST_PROCESS = FALSE - - -[SETOOLS_RUNNER] - -INPUT_MODULE = sextractor_runner_run_2 - -# Note: Make sure this doe not match the SExtractor background images -# (sexcat_background*) -FILE_PATTERN = sexcat - -NUMBERING_SCHEME = -0000000-0 - -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[PSFEX_RUNNER] - -# Use 80% sample for PSF model -FILE_PATTERN = star_split_ratio_80 - -NUMBERING_SCHEME = -0000000-0 - -# Path to executable for the PSF model (optional) -EXEC_PATH = psfex - -# Default psfex configuration file -DOT_PSFEX_FILE = $SP_CONFIG/default.psfex - - -[PSFEX_INTERP_RUNNER] - -# Use 20% sample for PSF validation -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat - -FILE_EXT = .psf, .fits, .cat - -NUMBERING_SCHEME = -0000000-0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = VALIDATION - -# Column names of position parameters -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# If True, measure and store ellipticity of the PSF (using moments) -GET_SHAPES = True - -# Minimum number of stars per CCD for PSF model to be computed -STAR_THRESH = 22 - -# Maximum chi^2 for PSF model to be computed on CCD -CHI2_THRESH = 2 diff --git a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg deleted file mode 100644 index 4e4164aaf..000000000 --- a/example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg +++ /dev/null @@ -1,24 +0,0 @@ --11.5 68 --6 186.5 -7 188 -10 64.5 -31 55 -50 38.5 -56.5 11.5 -188 8 -192 -4 -59.5 -11.5 -45 -33 -13.5 -64 -5 -154 --6 -155 --11 -64.5 --40 -44.5 --51.5 -30.5 --62.5 -22.5 --68 -9.5 --177 -2 --176 3 --78 12.5 --67.5 14.5 --38.5 50 diff --git a/example/cfis/mask_default/Messier_catalog.npy b/example/cfis/mask_default/Messier_catalog.npy deleted file mode 100644 index ef07eb032..000000000 Binary files a/example/cfis/mask_default/Messier_catalog.npy and /dev/null differ diff --git a/example/cfis/mask_default/Messier_catalog_updated.fits b/example/cfis/mask_default/Messier_catalog_updated.fits deleted file mode 100644 index 6a9f00096..000000000 Binary files a/example/cfis/mask_default/Messier_catalog_updated.fits and /dev/null differ diff --git a/example/cfis/mask_default/default.ww b/example/cfis/mask_default/default.ww deleted file mode 100644 index c2797f904..000000000 --- a/example/cfis/mask_default/default.ww +++ /dev/null @@ -1,40 +0,0 @@ -#--------------------------------- Weights ------------------------------------ - -WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) - -WEIGHT_MIN 0. # Pixel below those thresholds will be flagged -WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged -WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels - -#---------------------------------- Flags ------------------------------------- - -FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) - -FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels -FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs -FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits - -#---------------------------------- Polygons ---------------------------------- - -POLY_NAMES "" # Filename(s) of input DS9 regions -POLY_OUTFLAGS # FLAG values for polygon masks -POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks -POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? - -#---------------------------------- Output ------------------------------------ - -OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename -OUTFLAG_NAME flag.fits # Output FLAG-map filename - -#----------------------------- Miscellaneous --------------------------------- - -GETAREA N # Compute area for flags and weights (Y/N)? -GETAREA_WEIGHT 0.0 # Weight threshold for area computation -GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area -MEMORY_BUFSIZE 256 # Buffer size in lines -VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL -WRITE_XML N # Write XML file (Y/N)? -XML_NAME ww.xml # Filename for XML output -XSL_URL file:///usr/local/share/weightwatcher/ww.xsl - # Filename for XSL style-sheet -NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/example/cfis/mask_default/halo_mask.reg b/example/cfis/mask_default/halo_mask.reg deleted file mode 100644 index c44f25167..000000000 --- a/example/cfis/mask_default/halo_mask.reg +++ /dev/null @@ -1,50 +0,0 @@ - 274.66813 -1.25966 - 272.54579 32.47406 - 266.21222 65.67579 - 255.76731 97.82190 - 241.37579 128.40544 - 223.26462 156.94408 - 201.71942 182.98775 - 177.07997 206.12573 - 149.73486 225.99312 - 120.11532 242.27660 - 88.68848 254.71937 - 55.94996 263.12519 - 22.41606 267.36151 - -11.38436 267.36151 - -44.91826 263.12519 - -77.65678 254.71937 --109.08362 242.27660 --138.70315 225.99312 --166.04827 206.12573 --190.68772 182.98775 --212.23292 156.94408 --230.34409 128.40544 --244.73561 97.82190 --255.18052 65.67579 --261.51409 32.47406 --263.63643 -1.25966 --261.51409 -34.99339 --255.18052 -68.19511 --244.73561 -100.34123 --230.34409 -130.92476 --212.23292 -159.46341 --190.68772 -185.50708 --166.04827 -208.64506 --138.70315 -228.51245 --109.08362 -244.79593 - -77.65678 -257.23870 - -44.91826 -265.64452 - -11.38436 -269.88084 - 22.41606 -269.88084 - 55.94996 -265.64452 - 88.68848 -257.23870 - 120.11532 -244.79593 - 149.73486 -228.51245 - 177.07997 -208.64506 - 201.71942 -185.50708 - 223.26462 -159.46341 - 241.37579 -130.92476 - 255.76731 -100.34123 - 266.21222 -68.19511 - 272.54579 -34.99339 diff --git a/example/cfis/mask_default/ngc_cat.fits b/example/cfis/mask_default/ngc_cat.fits deleted file mode 100644 index f51546da7..000000000 Binary files a/example/cfis/mask_default/ngc_cat.fits and /dev/null differ diff --git a/example/cfis/star_selection.setools b/example/cfis/star_selection.setools index 8330a1eff..32197b466 100644 --- a/example/cfis/star_selection.setools +++ b/example/cfis/star_selection.setools @@ -1,4 +1,32 @@ ## SETools configuration file for star/galaxy separation based on size/mag properties +## +## ONE mask cut, and it is the instrument flags: +## IMAFLAGS_ISO == 0 the instrument flag image (bad columns, saturation) +## delivered with the exposure and read by SExtractor. +## +## That is deliberate, and it follows from what the two kinds of mask MEAN. +## An instrument flag marks a CORRUPTED MEASUREMENT — the pixels carry no +## usable signal — so a flagged star is not a star we could model badly, it is +## one we cannot model at all. The healsparse masks are sky-fixed LOCATION +## flags (a star halo, a manual region, a band with no data): they say where an +## object sits, not that its pixels are broken, so whether one disqualifies a +## PSF star is a judgement, not a fact about the data. +## +## So the star selection deliberately starts from OUTLIER REJECTION ALONE for +## that judgement. mask_query still queries the external masks per detection +## and writes MASK_EXT into this catalogue — carried for transparency and +## measurement, so their effect on the star sample can be measured before it is +## imposed. Flag transparently, cut downstream. +## +## MASK_EXT is the configurable pickup if outlier rejection proves +## insufficient. To impose it, add +## +## MASK_EXT == 0 +## +## beside each IMAFLAGS_ISO line below — one line per mask block, and that is +## the whole change. Which maps reach MASK_EXT is mask_query's MASK_PATHS +## config, which ships commented out; SETools has no bitwise operators, so the +## bit selection happens there and this file only ever tests for zero. [MASK:preselect] MAG_AUTO > 0 diff --git a/example/cfis_image_sims/README.md b/example/cfis_image_sims/README.md index b308cd6a8..fa0e00ac4 100644 --- a/example/cfis_image_sims/README.md +++ b/example/cfis_image_sims/README.md @@ -18,7 +18,10 @@ those cases are documented in the last column below and, at more length, under Every row is derived from the two bash scripts. "Module(s)" is the ShapePipe runner(s) the selected `.ini` names; "`.ini` selected" is what `job_sp_canfar_v2.0.bash` picks for that bit under sim settings -(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`, `star_cat_for_mask=onthefly`). +(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`). Bit 32 (mask exposures) is +gone: ShapePipe generates no masks (PR #847), so the bash scripts' +`star_cat_for_mask` setting and the `config_*_Ma_*.ini` configs it selected no +longer exist. | Bit | Stage | Module(s) | `.ini` selected (sim settings) | Sim special-casing | |----:|-------|-----------|--------------------------------|--------------------| @@ -27,10 +30,9 @@ runner(s) the selected `.ini` names; "`.ini` selected" is what | 4 | find exposures | `find_exposures_runner` | `config_tile_Fe.ini` | — | | 8 | retrieve exposure images | `get_images_runner` | `config_exp_Gie_symlink.ini` | symlink retrieval; completeness check expects 3 files vs. 6 for data | | 16 | split exposures, merge WCS headers | `split_exp_runner` | `config_exp_Sp.ini` | — | -| 32 | mask exposures | `mask_runner` | `config_exp_Ma_onthefly.ini` | — | | 64 | exposure PSF model | *(none — placeholder)* | *(none)* | **Placeholder.** For data this runs full exposure PSF modelling. For sims run_job writes a placeholder log and does nothing here; the sim PSF (`fake_psf_runner`) actually runs inside bit 512 | | 128 | merge exposure WCS headers → tile sqlite log | `merge_headers_runner` | `config_tile_Mh_exp.ini` | — | -| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx_nomask.ini` | `tile_det` is forced to `sx`, so the SExtractor-no-mask branch is always taken (the `uc` external-catalogue branch is never reached for sims) | +| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx.ini` | `tile_det` is forced to `sx`, so the SExtractor branch is always taken (the `uc` external-catalogue branch is never reached for sims). Tiles carry no flag image — ShapePipe generates no masks — so this runs with `FLAG_IMAGE = False` | | 512 | fake PSF + postage stamps | `fake_psf_runner`, then `vignetmaker_runner` ×2 | `config_exp_psfex.ini` (fake PSF), then `config_tile_PiViVi_canfar_sx.ini` (vignets) | **Two sub-runs.** run_job first calls the job script with `-j 64` → `config_exp_psfex.ini`, which despite its name runs `fake_psf_runner` (needs the sexcat from bit 256; run dir `run_sp_tile_fpsf`), then `-j 512` → `config_tile_PiViVi_canfar_sx.ini` for the two `vignetmaker_runner` runs. Data instead runs `psfex_interp_runner` + vignets here | | 1024 | multi-epoch shape measurement | `ngmix_runner` | `config_tile_Ng_batch_psfex_sx.ini` | — | | 2048 | create final catalogue | `make_cat_runner` | `config_tile_Mc_psfex.ini` | — | diff --git a/example/cfis_image_sims/config_exp_Ma_onthefly.ini b/example/cfis_image_sims/config_exp_Ma_onthefly.ini deleted file mode 100644 index df0307a19..000000000 --- a/example/cfis_image_sims/config_exp_Ma_onthefly.ini +++ /dev/null @@ -1,76 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = last:split_exp_runner - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask_simu - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis_image_sims/config_onthefly.mask_simu b/example/cfis_image_sims/config_onthefly.mask_simu deleted file mode 100644 index 1a63cc2e5..000000000 --- a/example/cfis_image_sims/config_onthefly.mask_simu +++ /dev/null @@ -1,86 +0,0 @@ -# Mask module configuration file for single-exposure images - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - - -## Border mask -[BORDER_PARAMETERS] - -BORDER_MAKE = True - -BORDER_WIDTH = 50 -BORDER_FLAG_VALUE = 4 - - -## Halo mask -[HALO_PARAMETERS] - -HALO_MAKE = False - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction spike mask -[SPIKE_PARAMETERS] - -SPIKE_MAKE = False - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier mask -[MESSIER_PARAMETERS] - -MESSIER_MAKE = False - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = False - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -TEMP_DIRECTORY = .temp - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis_image_sims/config_tile_Ma_onthefly.ini b/example/cfis_image_sims/config_tile_Ma_onthefly.ini deleted file mode 100644 index 0f49eccea..000000000 --- a/example/cfis_image_sims/config_tile_Ma_onthefly.ini +++ /dev/null @@ -1,82 +0,0 @@ -# ShapePipe configuration file for masking of tiles - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Ma - -# Add date and time to RUN_NAME, optional, default: False -; RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask tiles -[MASK_RUNNER] - -# Input directory, containing input files, single string or list of names -INPUT_DIR = run_sp_Git:get_images_runner, last:uncompress_fits_runner - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# Input file pattern(s), list of strings with length matching number of expected input file types -# Cannot contain wild cards -FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight - -# FILE_EXT (optional) list of string extensions to identify input files -FILE_EXT = .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask_simu - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = False - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -USE_EXT_STAR = False - -# File name suffix for the output flag files (optional) -PREFIX = pipeline diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini index 28ed565af..26564e4fa 100644 --- a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -130,7 +130,7 @@ ME_IMAGE_PATTERN = flag, image, weight [VIGNETMAKER_RUNNER_RUN_3] # Cut per-object coadd-frame segmentation stamps from the tile SExtractor -# SEGMENTATION check image (config_tile_Sx_nomask.ini: CHECKIMAGE = BACKGROUND, +# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, # SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode # guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN # centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg diff --git a/example/cfis_image_sims/config_tile_Sx_nomask.ini b/example/cfis_image_sims/config_tile_Sx.ini similarity index 91% rename from example/cfis_image_sims/config_tile_Sx_nomask.ini rename to example/cfis_image_sims/config_tile_Sx.ini index a5c12771c..78bcfb646 100644 --- a/example/cfis_image_sims/config_tile_Sx_nomask.ini +++ b/example/cfis_image_sims/config_tile_Sx.ini @@ -1,4 +1,9 @@ # ShapePipe configuration file for tile detection +# +# No flag image: ShapePipe generates no tile masks, and tiles have no +# instrument flag image of their own. Sky-fixed masks reach the catalogue as +# MASK_ columns, queried per object by make_cat. Hence +# default_noimaflags.param and FLAG_IMAGE = False below. ## Default ShapePipe options diff --git a/example/cfis_image_sims/config_tile_onthefly.mask_simu b/example/cfis_image_sims/config_tile_onthefly.mask_simu deleted file mode 100644 index 2e404ed96..000000000 --- a/example/cfis_image_sims/config_tile_onthefly.mask_simu +++ /dev/null @@ -1,89 +0,0 @@ -# Mask module config file for tiles - -## Paths to executables -[PROGRAM_PATH] - -WW_PATH = weightwatcher -WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww - -# Indicate cds client executable if no external star catalogue is available -# (e.g. no internet access on run nodes) -CDSCLIENT_PATH = findgsc2.2 - -## Border parameters -[BORDER_PARAMETERS] - -BORDER_MAKE = False - -BORDER_WIDTH = 1 -BORDER_FLAG_VALUE = 4 - - -## Halo parameters -[HALO_PARAMETERS] - -HALO_MAKE = False - -HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg -HALO_MAG_LIM = 13. -HALO_SCALE_FACTOR = 0.05 -HALO_MAG_PIVOT = 13.8 -HALO_FLAG_VALUE = 2 -HALO_REG_FILE = halo.reg - - -## Diffraction pike parameters -[SPIKE_PARAMETERS] - -SPIKE_MAKE = False - -SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg -SPIKE_MAG_LIM = 18. -SPIKE_SCALE_FACTOR = 0.3 -SPIKE_MAG_PIVOT = 13.8 -SPIKE_FLAG_VALUE = 128 -SPIKE_REG_FILE = spike.reg - - -## Messier parameters -[MESSIER_PARAMETERS] - -MESSIER_MAKE = False - -MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits -MESSIER_SIZE_PLUS = 0. -MESSIER_FLAG_VALUE = 16 - -## NGC mask -[NGC_PARAMETERS] - -NGC_MAKE = False - -NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits -NGC_SIZE_PLUS = 0. -NGC_FLAG_VALUE = 32 - - -## External flag -[EXTERNAL_FLAG] - -EF_MAKE = False - - -## Missing data parameters -[MD_PARAMETERS] - -MD_MAKE = False - -MD_THRESH_FLAG = 0.3 -MD_THRESH_REMOVE = 0.75 -MD_REMOVE = False - - -## Other parameters -[OTHER] - -KEEP_REG_FILE = False -KEEP_INDIVIDUAL_MASK = False - -TEMP_DIRECTORY = .temp_tiles diff --git a/pyproject.toml b/pyproject.toml index 8b259deec..47ce2b2d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,11 +19,12 @@ requires-python = ">=3.12" # the code actually requires the newer API. dependencies = [ "astropy>=7.0", # major 6 → 7 - "astroquery", "canfar", "cs_util>=0.2.1", "galsim>=2.8", "h5py", + "healsparse", + "hpgeom", "joblib>=1.4", "matplotlib>=3.10", "mccd>=1.2.4", @@ -36,7 +37,6 @@ dependencies = [ "python-pysap>=0.3", "PyQt5", "pyqtgraph", - "reproject>=0.19", "sf_tools>=2.0.4", "skaha>=1.7", "sqlitedict>=2.0", diff --git a/scripts/README.rst b/scripts/README.rst index 1d3e265a6..2b8067cb0 100644 --- a/scripts/README.rst +++ b/scripts/README.rst @@ -9,7 +9,6 @@ Python scripts ============== 1. `create_log_exp_headers`_ -2. `create_star_cat`_ create_log_exp_headers ====================== @@ -18,11 +17,3 @@ This as to run after the module `split_exp_runner` it will create a master log file containing all the WCS information for each CCDs of each single exposures. To run the script : `python create_log_exp_headers.py path/to/split_exp_runner/output path/to/srcipt/output_dir` - -create_star_cat -=============== - -This script create all the star catalogs required to run the mask module for a -computational node without internet access. -To run the script : -`python create_star_cat.py path/to/image_dir path/to/script/output_dir` diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py index 62fbf0564..cb9afcb0b 100755 --- a/scripts/python/canfar_avail_results.py +++ b/scripts/python/canfar_avail_results.py @@ -113,13 +113,6 @@ def parse_options(p_def): action="store_true", help="only check final catalogues", ) - parser.add_option( - "-m", - "--mask_only", - dest="mask_only", - action="store_true", - help="only check mask files (pipeline_flag)", - ) parser.add_option( "-x", "--extension", @@ -159,10 +152,6 @@ def check_options(options): print("Invalid PSF model '{}'".format(options.psf)) return False - if options.final_only and options.mask_only: - print("One one of the options '-f' or '-m' can be given") - return False - return True @@ -379,13 +368,10 @@ def main(argv=None): if param.final_only: result_base_names = ["final_cat"] - elif param.mask_only: - result_base_names = ["pipeline_flag"] else: result_base_names = [] types = [ "final_cat", - "pipeline_flag", "logs", "setools_mask", "setools_stat", diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py deleted file mode 100755 index 05e3d3bb9..000000000 --- a/scripts/python/create_star_cat.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -# -*- coding: utf-8 -*- - -"""Script create_star_cat.py - -:Description: Create reference star catalogue for masking of -bright star halos and diffraction spikes - -:Authors: Axel Guinot, Martin Kilbinger - -""" - - -import os -import re -import sys - -from cs_util import args as cs_args -from cs_util import logging as cs_logging - -from astropy.io import fits - -from shapepipe.utilities.file_io import write_atomic -from shapepipe.utilities.focal_plane import ccd_center_and_radius, focal_plane_disc -from shapepipe.utilities.vizier import query_vizier as _query_vizier - - -# GSC 2.3 catalog ID -CDS_CAT_ID = "I/305/out" - - -def query_vizier(ra, dec, radius_arcmin): - return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) - - -def main(input_dir, output_dir, kind): - - file_list = os.listdir(input_dir) - - for f in file_list: - if "image" not in f: - continue - - img_number = re.split("image", os.path.splitext(f)[0])[1] - fpath = os.path.join(input_dir, f) - - output_name = f"{output_dir}/star_cat{img_number}.fits" - if os.path.isfile(output_name): - continue - - if kind == "exp": - # One query covering the full MegaCam focal plane - ra, dec, radius_deg = focal_plane_disc(fpath) - radius = radius_deg * 60.0 - print( - f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" - ) - else: - # A single image: its own centre and half-diagonal. - ra, dec, radius_deg = ccd_center_and_radius(fits.getheader(fpath, 0)) - radius = radius_deg * 60.0 - - table = query_vizier(ra, dec, radius) - write_atomic(table, output_name) - - return 0 - - -def params_default(): - """Return default parameters, short options, types, and help strings.""" - _params = { - "input_dir": ".", - "output_dir": ".", - "kind": "exp", - } - _short_options = { - "input_dir": "-i", - "output_dir": "-o", - "kind": "-k", - } - _types = {} - _help_strings = { - "input_dir": "input directory containing image files; default is {}", - "output_dir": "output directory for star catalogues; default is {}", - "kind": "processing kind, 'exp' for full MegaCam focal plane, 'tile' for single image; default is {}", - } - return _params, _short_options, _types, _help_strings - - -if __name__ == "__main__": - - _params, _short_options, _types, _help_strings = params_default() - - options = cs_args.parse_options( - _params, - _short_options, - _types, - _help_strings, - ) - - cs_logging.log_command(sys.argv) - - main(options["input_dir"], options["output_dir"], options["kind"]) diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 24e72693a..946df5a8e 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -16,6 +16,7 @@ from sqlitedict import SqliteDict from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query def get_output_name(output_dir, file_number_string): @@ -209,6 +210,68 @@ def save_sm_data( return n_obj +def parse_mask_ext_paths(paths_str): + """Parse Mask Ext Paths. + + Parse the ``MASK_EXT_PATHS`` config value into a ``band -> path`` mapping. + + Parameters + ---------- + paths_str : str + Comma-separated ``band:path`` pairs, e.g. + ``u:/path/mask_u.hsp, g:/path/mask_g.hsp`` + + Returns + ------- + dict + Mapping from band name to healsparse map path + + """ + band_paths = {} + for pair in paths_str.split(","): + band, path = pair.split(":", 1) + band_paths[band.strip()] = path.strip() + + return band_paths + + +def save_mask_ext_data(final_cat_file, band_paths, w_log): + """Save External Mask Data. + + Query per-band external healsparse masks at each object's world position + and write one ``MASK_`` column per band into the final catalogue. + Object positions are read from the SExtractor windowed world coordinates + (``XWIN_WORLD`` = RA, ``YWIN_WORLD`` = Dec, both in degrees) carried in the + ``RESULTS`` extension. Objects falling outside a map's coverage receive + that map's sentinel value (``healsparse.HealSparseMap.get_values_pos`` + returns the map's sentinel — ``-1`` for integer maps — verbatim), which is + the documented off-map flag. + + The lookup itself is ``shapepipe.utilities.mask_query.query_map``, + shared with the ``mask_query`` module: one primitive, two consumers. + + Parameters + ---------- + final_cat_file : file_io.FITSCatalogue + Final catalogue + band_paths : dict + Mapping from band name to healsparse map path + w_log : logging.Logger + Logging instance + + """ + final_cat_file.open() + ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"]) + dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"]) + + for band, path in band_paths.items(): + w_log.info(f"Query external mask for band {band}: {path}") + values = mask_query.query_map(path, ra, dec) + final_cat_file.add_col(f"MASK_{band}", values) + + final_cat_file.close() + + class SaveCatalogue: """Save Catalogue. diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 307dc2ffe..176341757 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -137,4 +137,14 @@ def make_cat_runner( if save_psf: err_msg = sc_inst.process("psf", galaxy_psf_path) + # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38): + # add one MASK_ column per band, queried at each object's world + # position. Absent config is a strict no-op. + if config.has_option(module_config_sec, "MASK_EXT_PATHS"): + band_paths = make_cat.parse_mask_ext_paths( + config.getexpanded(module_config_sec, "MASK_EXT_PATHS") + ) + w_log.info("Save external mask data") + make_cat.save_mask_ext_data(final_cat_file, band_paths, w_log) + return None, None diff --git a/src/shapepipe/modules/mask_package/__init__.py b/src/shapepipe/modules/mask_package/__init__.py deleted file mode 100644 index bdfece65b..000000000 --- a/src/shapepipe/modules/mask_package/__init__.py +++ /dev/null @@ -1,185 +0,0 @@ -"""MASK MODULE. - -This package contains the module for ``mask``. - -:Author: Axel Guinot - -:Parent module: ``split_exp_runner`` or None - -:Input: Single-exposure single-CCD image, weight file, flag file (optional), - and star catalogue (optional) - -:Output: Single-exposure single-CCD flag files - -Description -=========== - -This module creates masks for bright stars, diffraction spikes, deep sky -objects (from the Messier and NGC catalogues), borders, and other artifacts. If -a flag file is given as input, for example from pre-processing, the mask that -is created by this module is joined with the mask from this external flag file. -In this case the config flag ``USE_EXT_FLAG`` needs to be set to ``True``. To -distinguish the newly created output flag file from the input ones, a prefix -can added as specificed by the config entry ``PREFIX``. - -An NGC catalogue with positions, sizes, and types is provided with -``shapepipe``, -`source `_. - -Masked pixels of different mask types are indicated by integers, which -conveniently are powers of two such that they can be combined bit-wise. - -To mask bright stars, this module either creates a star catalogue from the -online -`guide star catalogue `_ -database relevant to the the footprint. This is done by calling a CDs -(Centre de Données astronomique de Strasbourg) -`client program `_. -Note that this requires online access, -which in some cases is not granted on compute nodes of a cluster. In this case, -set the config flag ``USE_EXT_STAR = False``. Alternatively, a star -catalogue can be created before running this module via the script -``create_star_cat``. During the processing of this module, this star catalogue -is read from disk, with ``USE_SET_STAR = True``. - -The masking is done with the software ``WeightWatcher`` :cite:`marmo:08`, -which is installed by ``ShapePipe`` by default. - -Module-specific config file entries -=================================== - -USE_EXT_FLAG : bool - Use external flag file to join with the mask created here; - if ``True`` flag file needs to be given on input -USE_EXT_STAR : bool - Read external star catalogue instead of creating one during the - call of this module; - if ``True`` star catalogue file needs to be given on input -MASK_CONFIG_PATH : str - Path to mask config file -HDU : int, optional - HDU of external flag FITS file; the default value is ``0`` -PREFIX : str, optional - Prefix to be appended to output file name ``flag``; - helps to distinguish the file patterns of newly created and external - mask files -CHECK_EXISTING_DIR : str, optional - If given, search this directory for existing mask files; the - corresponding images will then not be processed - -Mask config file -================ - -An additional configuration file is used by the mask module, its path is -``MASK_CONFIG_PATH`` in the module config section, see above. The following -describes the config file sections and their entries. - -[PROGRAM_PATH] --------------- - -WW_PATH : str, optional - Full path to the WeightWatcher executable (``ww``) on the system ; if not - set the version controlled WeightWatcher installation in the ShapePipe - environment will be used -WW_CONFIG_FILE : str - Path to the WeightWatcher configuration file -CDSCLIENT_PATH : str, optional - Path to CDS client executable; required if ``USE_EXT_STAR = False`` - -[BORDER_PARAMETERS] -------------------- - -BORDER_MAKE : bool - Create mask around borders if ``True`` -BORDER_WIDTH : int - Width of border mask in pixels -BORDER_FLAG_VALUE : int - Border mask pixel value, power of 2 - -[HALO_PARAMETERS] ------------------ - -HALO_MAKE : bool - Create mask for halos of bright stars if ``True`` -HALO_MASKMODEL_PATH : str - Path to halo mask geometry (``.reg`` file) -HALO_MAG_LIM : float - Faint stellar magnitude limit for halo mask -HALO_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and halo mask size -HALO_MAG_PIVOT : float - Pivot stellar magnitude -HALO_FLAG_VALUE : int - Halo mask pixel value, power of 2 -HALO_REG_FILE : str - Output halo mask ``.reg`` file - -[SPIKE_PARAMETERS] ------------------- - -SPIKE_MAKE : bool - Create mask for diffraction spikes of bright stars if ``True`` -SPIKE_MASKMODEL_PATH : str - Path to diffraction spike geometry (``.reg`` file) -SPIKE_MAG_LIM : - Faint stellar magnitude limit for spike mask -SPIKE_SCALE_FACTOR : float - Factor to scale between magnitude (relative to pivot) and spike mask size -SPIKE_MAG_PIVOT : float - Pivot stellar magnitude -SPIKE_FLAG_VALUE : int - Diffraction spike pixel value, power of two -SPIKE_REG_FILE : str - Output spike mask ``.reg`` file - -[MESSIER_PARAMETERS] --------------------- - -MESSIER_MAKE : bool - Create mask around Messier objects if ``True`` -MESSIER_CAT_PATH : str - Path to Messier catalogue -MESSIER_SIZE_PLUS : float - Fraction to increase Messier mask -MESSIER_FLAG_VALUE : int - Messier mask pixel value, power of 2 - -[NGC_PARAMETERS] --------------------- - -NGC_MAKE : bool - Create mask around NGC objects if ``True`` -NGC_CAT_PATH : str - Path to NGC catalogue -NGC_SIZE_PLUS : float - Fraction to increase NGC mask -NGC_FLAG_VALUE : int - NGC mask pixel value, power of 2 - -[MD_PARAMETERS] ---------------- - -MD_MAKE : bool - Account for missing data (zero-valued pixels) if ``True`` -MD_THRESH_FLAG : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked as flagged -MD_THRESH_REMOVE : float - Threshold; if relative number of missing data is larger than this - threshold, image is marked for removal -MD_REMOVE : bool - Image is removed if marked for removal - -[OTHER] -------- - -TEMP_DIRECTORY : str - Path to temporary dictionary -KEEP_INDIVIDUAL_MASK : bool - Keep individual masks in addition to merged mask file -KEEP_REG_FILE : bool - Keep ``.reg`` mask file - -""" - -__all__ = ["mask"] diff --git a/src/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py deleted file mode 100644 index ba6d586ad..000000000 --- a/src/shapepipe/modules/mask_package/mask.py +++ /dev/null @@ -1,1271 +0,0 @@ -"""MASK. - -This module contains a class to create star mask for an image. - -:Authors: Axel Guinot, Martin Kilbinger - -""" - -import os -import re - -import numpy as np -from astropy import units, wcs -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.table import Table - -from shapepipe.pipeline import file_io -from shapepipe.pipeline.config import CustomParser -from shapepipe.pipeline.execute import execute -from shapepipe.utilities.file_system import mkdir -from shapepipe.utilities.vizier import query_vizier - - -class Mask(object): - """Mask. - - Class to create mask based on a star catalogue. - - Parameters - ---------- - image_path : str - Path to image (FITS format) - weight_path : str - Path to the weight image (FITS format) - image_prefix : str - Prefix to input image name, specify as ``'none'`` for no prefix - image_num : str - File number identified - config_filepath : str - Path to the ``.mask`` config file - output_dir : str - Path to the output directory - w_log : logging.Logger - Log file - path_external_flag : str, optional - Path to external flag file, default is ``None`` (not used) - outname_base : str, optional - Output file name base, default is ``flag`` - check_existing_dir : str, optional - If not ``None`` (default), search path for existing mask files - star_cat_path : str, optional - Path to external star catalogue, default is ``None`` (not used; - instead the star catalogue is produced on the fly at run time) - hdu : int, optional - HDU number, default is ``0`` - - """ - - def __init__( - self, - image_path, - weight_path, - image_prefix, - image_num, - config_filepath, - output_dir, - w_log, - path_external_flag=None, - outname_base="flag", - check_existing_dir=None, - star_cat_path=None, - hdu=0, - ): - - # Path to the image to mask - self._image_fullpath = image_path - - # Path to the weight associated to the image - self._weight_fullpath = weight_path - - # Input image prefix - if (image_prefix.lower() != "none") and (image_prefix != ""): - self._img_prefix = f"{image_prefix}_" - else: - self._img_prefix = "" - - # File number identified - self._img_number = image_num - - # Path to mask config file - self._config_filepath = config_filepath - - # Path to the output directory - self._output_dir = output_dir - - # Log file - self._w_log = w_log - - # Path to an external flag file - self._path_external_flag = path_external_flag - - # Output file base name - self._outname_base = outname_base - - # Search path for existing mask files - self._check_existing_dir = check_existing_dir - - # Set external star catalogue path if given - if star_cat_path is not None: - self._star_cat_path = star_cat_path - - self._hdu = hdu - - # Read mask config file - self._get_config() - - # Set parameters needed for the star detection - self._set_image_coordinates() - - # Set error flag - self._err = False - - # Guide Star Catalogue parameters - #self._CDS_cat_ID = "I/271/out" # GSC 2.2, does not have Fmag - self._CDS_cat_ID = "I/305/out" # GSC 2.3 - - # Keys in CDS astroquery result - self._cds_keys = ["GSC2.3", "RAJ2000", "DEJ2000", "Fmag", "jmag", "Vmag", "Nmag", "Class"] - - # Minimal scaling for halo and spike polygon templates - self._scaling_min = 0.1 - - def _get_config(self): - """Get Config. - - Read the config file and set parameters. - - Raises - ------ - ValueError - If config file name is ``None`` - IOError - If config file not found - - """ - if self._config_filepath is None: - raise ValueError("No path to config file given") - - if not os.path.exists(self._config_filepath): - raise IOError(f'Config file "{self._config_filepath}" not found') - - conf = CustomParser() - conf.read(self._config_filepath) - - self._config = { - "PATH": {}, - "BORDER": {}, - "HALO": {}, - "SPIKE": {}, - "MESSIER": {}, - "NGC": {}, - "MD": {}, - } - - if conf.has_option("PROGRAM_PATH", "WW_PATH"): - self._config["PATH"]["WW"] = conf.getexpanded( - "PROGRAM_PATH", "WW_PATH" - ) - else: - self._config["PATH"]["WW"] = "weightwatcher" - self._config["PATH"]["WW_configfile"] = conf.getexpanded( - "PROGRAM_PATH", "WW_CONFIG_FILE" - ) - if conf.has_option("PROGRAM_PATH", "CDSCLIENT_PATH"): - self._config["PATH"]["CDSclient"] = conf.getexpanded( - "PROGRAM_PATH", "CDSCLIENT_PATH" - ) - elif self._star_cat_path is not None: - self._config["PATH"]["star_cat"] = self._star_cat_path - else: - raise ValueError( - "Either [PROGRAM_PATH]:CDSCLIENT_PATH in the mask config file " - + " or a star catalogue as module input needs to be present" - ) - - self._config["PATH"]["temp_dir"] = self._get_temp_dir_path( - conf.getexpanded("OTHER", "TEMP_DIRECTORY") - ) - self._config["BORDER"]["make"] = conf.getboolean( - "BORDER_PARAMETERS", "BORDER_MAKE" - ) - if self._config["BORDER"]["make"]: - self._config["BORDER"]["width"] = conf.getint( - "BORDER_PARAMETERS", "BORDER_WIDTH" - ) - self._config["BORDER"]["flag"] = conf.get( - "BORDER_PARAMETERS", "BORDER_FLAG_VALUE" - ) - - for mask_shape in ["HALO", "SPIKE"]: - - self._config[mask_shape]["make"] = conf.getboolean( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAKE", - ) - self._config[mask_shape]["individual"] = conf.getboolean( - "OTHER", "KEEP_INDIVIDUAL_MASK" - ) - - if self._config[mask_shape]["make"]: - - self._config[mask_shape]["maskmodel_path"] = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MASKMODEL_PATH", - ) - self._config[mask_shape]["mag_lim"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_LIM", - ) - self._config[mask_shape]["scale_factor"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_SCALE_FACTOR", - ) - self._config[mask_shape]["mag_pivot"] = conf.getfloat( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_MAG_PIVOT", - ) - self._config[mask_shape]["flag"] = conf.getint( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_FLAG_VALUE", - ) - - if conf.getboolean("OTHER", "KEEP_REG_FILE"): - reg_file = conf.getexpanded( - f"{mask_shape}_PARAMETERS", - f"{mask_shape}_REG_FILE", - ) - self._config[mask_shape]["reg_file"] = ( - f'{self._config["PATH"]["temp_dir"]}/' - + f'{re.split(".reg", reg_file)[0]}' - + f"{self._img_number}.reg" - ) - else: - self._config[mask_shape]["reg_file"] = None - - for mask_type in ["MESSIER", "NGC"]: - - self._config[mask_type]["make"] = conf.getboolean( - f"{mask_type}_PARAMETERS", f"{mask_type}_MAKE" - ) - - if self._config[mask_type]["make"]: - self._config[mask_type]["cat_path"] = conf.getexpanded( - f"{mask_type}_PARAMETERS", - f"{mask_type}_CAT_PATH", - ) - self._config[mask_type]["size_plus"] = conf.getfloat( - f"{mask_type}_PARAMETERS", - f"{mask_type}_SIZE_PLUS", - ) - self._config[mask_type]["flag"] = conf.getint( - f"{mask_type}_PARAMETERS", - f"{mask_type}_FLAG_VALUE", - ) - - self._config["MD"]["make"] = conf.getboolean("MD_PARAMETERS", "MD_MAKE") - - if self._config["MD"]["make"]: - self._config["MD"]["thresh_flag"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_FLAG" - ) - self._config["MD"]["thresh_remove"] = conf.getfloat( - "MD_PARAMETERS", "MD_THRESH_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean( - "MD_PARAMETERS", "MD_REMOVE" - ) - self._config["MD"]["remove"] = conf.getboolean("MD_PARAMETERS", "MD_REMOVE") - - def _set_image_coordinates(self): - """Set Image Coordinates. - - Compute the image coordinates for matching with the star catalogue - and star mask. - - """ - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - self._header = img.get_header() - img_shape = img.get_data().shape - img.close() - del img - - self._wcs = wcs.WCS(self._header) - - # Compute field center - - # Note: get_data().shape corresponds to (n_y, n_x) - pix_center = [img_shape[1] / 2.0, img_shape[0] / 2.0] - wcs_center = self._wcs.all_pix2world([pix_center], 1)[0] - self._fieldcenter = {} - self._fieldcenter["pix"] = np.array(pix_center) - self._fieldcenter["wcs"] = SkyCoord( - ra=wcs_center[0], dec=wcs_center[1], unit="deg" - ) - - # Get the four corners of the image - corners = self._wcs.calc_footprint() - self._corners_sc = SkyCoord( - ra=corners[:, 0] * units.degree, - dec=corners[:, 1] * units.degree, - ) - - # Compute image radius = image diagonal - self._img_radius = self._get_image_radius() - - def make_mask(self): - """Make Mask. - - Main function to create the mask. - - """ - output_file_name = ( - f"{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - if os.path.exists(f"{self._check_existing_dir}//{output_file_name}"): - return None, None - - if self._config["MD"]["make"]: - self.missing_data() - - if self._config["HALO"]["make"] or self._config["SPIKE"]["make"]: - stars = self.find_stars( - np.array( - [ - self._fieldcenter["wcs"].ra.value, - self._fieldcenter["wcs"].dec.value, - ] - ), - radius=self._img_radius, - ) - - if not self._err: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._create_mask( - stars=stars, - types=_type, - mag_limit=self._config[_type]["mag_lim"], - scale_factor=self._config[_type]["scale_factor"], - mag_pivot=self._config[_type]["mag_pivot"], - ) - - if not self._err: - mask_name = [] - if self._config["HALO"]["make"] and self._config["SPIKE"]["make"]: - self._exec_WW(types="ALL") - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}halo_spike_flag' - + f"{self._img_number}.fits" - ) - mask_name.append(None) - else: - for _type in ("HALO", "SPIKE"): - if self._config[_type]["make"]: - self._exec_WW(types=_type) - mask_name.append( - f'{self._config["PATH"]["temp_dir"]}' - + f"{_type.lower()}_flag{self._img_number}.fits" - ) - else: - mask_name.append(None) - - masks_internal = {} - if not self._err: - if self._config["BORDER"]["make"]: - masks_internal["BORDER"] = self.mask_border( - width=self._config["BORDER"]["width"] - ) - - if not self._err: - for _type in ("MESSIER", "NGC"): - if self._config[_type]["make"]: - masks_internal[_type] = self.mask_dso( - self._config[_type]["cat_path"], - size_plus=self._config[_type]["size_plus"], - flag_value=self._config[_type]["flag"], - obj_type=_type, - ) - - if not self._err: - try: - im_pass = self._config["MD"]["im_remove"] - except Exception: - im_pass = True - - if not self._err: - path_external_flag = self._path_external_flag - - if not self._err: - if im_pass: - final_mask = self._build_final_mask( - path_mask1=mask_name[0], - path_mask2=mask_name[1], - masks_internal=masks_internal, - path_external_flag=path_external_flag, - ) - - if not self._config["HALO"]["individual"]: - if mask_name[0] is not None: - self._rm_fits1_stdout, self._rm_fits1_stderr = execute( - f"rm {mask_name[0]}" - ) - if mask_name[1] is not None: - self._rm_fits2_stdout, self._rm_fits2_stderr = execute( - f"rm {mask_name[1]}" - ) - - output_file_name = ( - f"{self._output_dir}/{self._img_prefix}" - + f"{self._outname_base}{self._img_number}.fits" - ) - - self._mask_to_file( - input_mask=final_mask, - output_fullpath=output_file_name, - ) - - # Handle stdout / stderr - # _CDS_stdout/_CDS_stderr are only set when find_stars ran, i.e. - # when HALO_MAKE or SPIKE_MAKE is True (False for image sims) - general_stdout = "" - general_stderr = "" - if hasattr(self, "_CDS_stdout"): - general_stdout += f"\nCDSClient\n{self._CDS_stdout}" - if self._CDS_stderr != "": - general_stderr += f"\nCDSClient\n{self._CDS_stderr}" - if hasattr(self, "_WW_stdout") or hasattr(self, "_WW_stdout"): - general_stdout += f"\n\nWeightWatcher\n{self._WW_stdout}" - if self._WW_stderr != "": - general_stderr += f"\n\nWeightWatcher\n{self._WW_stderr}" - if hasattr(self, "_rm_reg_stderr") or hasattr(self, "_rm_reg_stdout"): - general_stdout += f"\n\nrm reg file\n{self._rm_reg_stdout}" - if self._rm_reg_stderr != "": - general_stderr += f"\n\nrm reg file\n{self._rm_reg_stderr}" - if hasattr(self, "_rm_fits1_stderr") or hasattr( - self, "_rm_fits1_stdout" - ): - general_stdout += f"\n\nrm fits1 file\n{self._rm_fits1_stdout}" - if self._rm_fits1_stderr != "": - general_stderr += f"\n\nrm fits1 file\n{self._rm_fits1_stderr}" - if hasattr(self, "_rm_fits2_stderr") or hasattr( - self, "_rm_fits2_stdout" - ): - general_stdout += f"\n\nrm fits2 file\n{self._rm_fits2_stdout}" - if self._rm_fits2_stderr != "": - general_stderr += f"\n\nrm fits2 file\n{self._rm_fits2_stderr}" - - return general_stdout, general_stderr - - def find_stars(self, position, radius): - """Find Stars. - - Return GSC (Guide Star Catalog) objects for a field with center - (RA, Dec) and radius :math:`r`. - - Parameters - ---------- - position : numpy.ndarray - Position of the center of the field - radius : float - Radius in which the query is done (in arcmin) - - Returns - ------- - dict - Star dictionnary for GSC objects in the field - - Raises - ------ - ValueError - For invalid configuration options - - """ - if "star_cat" in self._config["PATH"]: - self._CDS_stdout = Table.read(self._config["PATH"]["star_cat"]) - else: - # For some exposures, Vizier returned empty star list if input position - # is not single (? or double) precision - p = np.array(position, dtype='double') - - coord = SkyCoord(ra=p[0] * units.deg, dec=p[1] * units.deg, frame="icrs") - - self._CDS_stdout = query_vizier(p[0], p[1], radius, self._CDS_cat_ID) - - self._CDS_stderr = "" - - return self._make_star_cat(self._CDS_stdout) - - def mask_border(self, width=100, flag_value=4): - """Create Mask Border. - - Mask ``width`` pixels around the image. - - Parameters - ---------- - width : int - Width of the mask mask border - flag_value : int - Value of the flag for the border (power of 2) - - Returns - ------- - numpy.ndarray - Array containing the mask - - Raises - ------ - ValueError - If ``width`` is ``None`` - - """ - if width is None: - raise ValueError("Width for border mask not provided") - - # Note that python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - flag[0:width, :] = flag_value - flag[-width:, :] = flag_value - flag[:, 0:width] = flag_value - flag[:, -width:] = flag_value - - return flag - - def mask_dso( - self, - cat_path, - size_plus=0.1, - flag_value=8, - obj_type="Messier", - ): - """Mask DSO. - - Create a circular patch for deep-sky objects (DSOs), e.g. - Messier or NGC objects. - - Parameters - ---------- - cat_path : str - Path to the deep-sky catalogue - size_plus : float - Increase the size of the mask by this factor - (e.g. ``0.1`` means 10%) - flag_value : int - Value of the flag, some power of 2 - obj_type : {'Messier', 'NGO'}, optional - Object type - - Returns - ------- - numpy.ndarray or ``None`` - If no deep-sky objects are found in the field return ``None`` and - the flag map - - Raises - ------ - ValueError - If ``size_plus`` is negative - ValueError - If ``cat_path`` is ``None`` - - """ - if size_plus < 0: - raise ValueError( - "deep-sky mask size increase variable cannot be negative" - ) - - if cat_path is None: - raise ValueError("Path to deep-sky object catalogue not provided") - - m_cat, header = fits.getdata(cat_path, header=True) - - unit_ra = file_io.get_unit_from_fits_header(header, "ra") - unit_dec = file_io.get_unit_from_fits_header(header, "dec") - m_sc = SkyCoord( - ra=m_cat["ra"] * unit_ra, - dec=m_cat["dec"] * unit_dec, - ) - - unit_size_X = file_io.get_unit_from_fits_header(header, "size_X") - unit_size_Y = file_io.get_unit_from_fits_header(header, "size_Y") - - # Loop through all deep-sky objects and check whether the object's - # disc overlaps the image footprint - indices = [] - size_max_deg = [] - for idx, m_obj in enumerate(m_cat): - - # DSO size - # r = max(m_obj['size']) * units.arcmin - r = max( - m_obj["size_X"] * unit_size_X, - m_obj["size_Y"] * unit_size_Y, - ) - r_deg = r.to(units.degree) - size_max_deg.append(r_deg) - - # Add index to list if the DSO disc overlaps the image: - # distance between DSO centre and image centre smaller than - # DSO radius plus image half-diagonal. (Testing only the image - # corners against the DSO radius, as done previously, misses - # objects that are smaller than the image and lie away from - # the corners.) - dist = self._fieldcenter["wcs"].separation(m_sc[idx]) - if dist < r_deg + self._img_radius * units.arcmin: - indices.append(idx) - - self._w_log.info( - f"Found {len(indices)} {obj_type} objects overlapping with" " image" - ) - - if len(indices) == 0: - # No closeby deep-sky object found - return None - - # Compute number of DSO center coordinates in footprint, for logging - # purpose only - n_dso_center_in_footprint = 0 - for idx in indices: - in_img = self._wcs.footprint_contains(m_sc[idx]) - self._w_log.info( - "(obj_type, ra, dec, in_img) = " - + f"({obj_type}, " - + f'{m_cat["ra"][idx]}, ' - + f'{m_cat["dec"][idx]}, ' - + f"{in_img})" - ) - - # Note: python image array is [y, x] - flag = np.zeros( - ( - int(self._fieldcenter["pix"][1] * 2), - int(self._fieldcenter["pix"][0] * 2), - ), - dtype="uint16", - ) - - nx = self._fieldcenter["pix"][0] * 2 - ny = self._fieldcenter["pix"][1] * 2 - for idx in indices: - m_center = np.hstack( - self._wcs.all_world2pix( - m_cat["ra"][idx], - m_cat["dec"][idx], - 0, - ) - ) - r_pix = ( - size_max_deg[idx].to(units.deg).value - * (1 + size_plus) - / np.abs(self._wcs.pixel_scale_matrix[0][0]) - ) - - # The following accounts for deep-sky centers outside of image, - # without creating masks for coordinates out of range - y_c, x_c = np.ogrid[0:ny, 0:nx] - mask_tmp = (x_c - m_center[0]) ** 2 + ( - y_c - m_center[1] - ) ** 2 <= r_pix**2 - - flag[mask_tmp] = flag_value - - return flag - - def missing_data(self): - """Find Missing Data. - - Look for zero-valued pixels in image. Flag if their relative number - is larger than a threshold. - """ - # Open image - img = file_io.FITSCatalogue(self._image_fullpath, hdu_no=0) - img.open() - - # Get total number of pixels - im_shape = img.get_data().shape - tot = float(im_shape[0] * im_shape[1]) - - # Compute number and ratio of missing data (zero-valued pixels) - missing = float(len(np.where(img.get_data() == 0.0)[0])) - self._ratio = missing / tot - - # Mark image as to be flagged if ratio larger than 'flag' threshold - if self._ratio >= self._config["MD"]["thresh_flag"]: - self._config["MD"]["im_flagged"] = True - else: - self._config["MD"]["im_flagged"] = False - - # Mark image as to be removed if flag is True and - # ratio large than 'remove' threshold. - # Reset all other mask 'make' flags to False (no other mask needs - # to be created) - if self._config["MD"]["remove"]: - if self._ratio >= self._config["MD"]["thresh_remove"]: - self._config["MD"]["im_remove"] = True - for idx in ["HALO", "SPIKE", "MESSIER", "BORDER"]: - self._config[idx]["make"] = False - else: - self._config["MD"]["im_remove"] = False - - img.close() - - def sphere_dist(self, position1, position2): - """Compute Spherical Distance. - - Compute spherical distance between 2 points. - - Parameters - ---------- - position1 : numpy.ndarray - [x,y] first point (in pixels) - position2 : numpy.ndarray - [x,y] second point (in pixels) - - Returns - ------- - float - The distance in degrees. - - Raises - ------ - ValueError - If input positions are not Numpy arrays - - """ - if ( - type(position1) is not np.ndarray - or type(position2) is not np.ndarray - ): - raise ValueError("Object coordinates need to be a numpy.ndarray") - - p1 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position1[0], position1[1], 1) - ) - p2 = (np.pi / 180.0) * np.hstack( - self._wcs.all_pix2world(position2[0], position2[1], 1) - ) - - dTheta = p1 - p2 - dLong = dTheta[0] - dLat = dTheta[1] - - dist = 2 * np.arcsin( - np.sqrt( - np.sin(dLat / 2.0) ** 2.0 - + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(dLong / 2.0) ** 2.0 - ) - ) - - return dist * (180.0 / np.pi) * 3600.0 - - def _get_image_radius(self, center=None): - """Get Image Radius. - - Compute the diagonal distance of the image in arcmin. - - Parameters - ---------- - center : numpy.ndarray, optional - Coordinates of the center of the image (in pixels) - - Returns - ------- - float - The diagonal distance of the image in arcmin - - Raises - ------ - TypeError - If centre is not a Numpy array - - """ - if center is None: - return ( - self.sphere_dist(self._fieldcenter["pix"], np.zeros(2)) / 60.0 - ) - - else: - if isinstance(center, np.ndarray): - return self.sphere_dist(center, np.zeros(2)) / 60.0 - else: - raise TypeError( - "Image center coordinates has to be a numpy.ndarray" - ) - - def _make_star_cat(self, CDSclient_output): - """Make Star Catalogue. - - Create a dictionary from an astroquery request. - - Parameters - ---------- - CDSclient_output : str - Output astroquery - - Returns - ------- - dict - Star dictionary containing all information - - """ - header = [] - stars = {} - - for key in self._cds_keys: - stars[key] = CDSclient_output[key] - - return stars - - def _create_mask( - self, - stars, - types="HALO", - mag_limit=18.0, - mag_pivot=13.8, - scale_factor=0.3, - ): - """Create Mask. - - Apply mask from model to stars and save into DS9 region file. - - Parameters - ---------- - stars : dict - Stars dictionary (output of ``find_stars``) - types : {'HALO', 'SPIKE'}, optional - Type of mask, options are ``HALO`` or ``SPIKE`` - mag_limit : float, optional - Faint magnitude limit for mask, default is ``18.0`` - mag_pivot : float, optional - Pivot magnitude for the model, default is ``13.8`` - scale_factor : float, optional - Scaling for the model, default is ``0.3`` - - Raises - ------ - ValueError - If no star catalogue is provided - ValueError - If an invalid option is provided for type - - """ - if stars is None: - raise ValueError("Star catalogue dictionary not provided") - - if types not in ("HALO", "SPIKE"): - raise ValueError('Mask types need to be in ["HALO", "SPIKE"]') - - if self._config[types]["reg_file"] is None: - reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - else: - reg = self._config[types]["reg_file"] - - mask_model = np.loadtxt( - self._config[types]["maskmodel_path"] - ).transpose() - mask_reg = open(reg, "w") - - stars_used = [[], [], []] - - """ - star_zip = zip( - stars["RA(J2000)"], - stars["Dec(J2000)"], - stars["Fmag"], - stars["Jmag"], - stars["Vmag"], - stars["Nmag"], - stars["Clas"], - ) - """ - - # Get keys without object name - keys_to_use = self._cds_keys[1:] - star_zip = zip(*(stars[k] for k in keys_to_use)) - - for ra, dec, Fmag, Jmag, Vmag, Nmag, clas in star_zip: - # Compute mean magnitude over the available (finite) bands. - # Missing GSC bands are NaN and must be excluded: a single NaN - # would make the mean NaN and silently fail the - # ``mag < mag_limit`` test below, leaving bright stars with - # incomplete photometry (the ones that most need masking) - # unmasked. - mags = [ - band - for band in (Fmag, Jmag, Vmag, Nmag) - if band is not None and np.isfinite(band) - ] - if len(mags) > 0: - mag = sum(mags) / len(mags) - else: - mag = None - self._w_log.info( - f"No finite {types} magnitude for star at ra={ra} " - + f"dec={dec}; object not masked" - ) - - if ( - ra is not None - and dec is not None - and mag is not None - and clas is not None - ): - if (mag < mag_limit) and (clas == 0): - scaling = 1.0 - scale_factor * (mag - mag_pivot) - if scaling < self._scaling_min: - scaling = self._scaling_min - pos = self._wcs.all_world2pix(ra, dec, 0) - stars_used[0].append(pos[0]) - stars_used[1].append(pos[1]) - stars_used[2].append(scaling) - - for idx in range(len(stars_used[0])): - poly = "polygon(" - for x, y in zip(mask_model[0], mask_model[1]): - angle = np.arctan2(y, x) - ll = stars_used[2][idx] * np.sqrt(x**2 + y**2) - xnew = ll * np.cos(angle) - ynew = ll * np.sin(angle) - poly = ( - f"{poly}{str(stars_used[0][idx] + xnew + 0.5)} " - + f"{str(stars_used[1][idx] + ynew + 0.5)} " - ) - poly = f"{poly})\n" - mask_reg.write(poly) - - mask_reg.close() - - def _exec_WW(self, types="HALO"): - """Execute WeightWatcher. - - Execute WeightWatcher to transform ``.reg`` to ``.fits`` flag map. - - Parameters - ---------- - types : {'HALO', 'SPIKE', 'ALL'}, optional - Type of WeightWatcher execution, options are ``HALO``, - ``SPIKE`` or ``ALL`` - - Raises - ------ - BaseCatalogue.CatalogFileNotFound - If catalogue file not found - - """ - if types in ("HALO", "SPIKE"): - - default_reg = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}' - + f"{self._img_number}.reg" - ) - default_out = ( - f'{self._config["PATH"]["temp_dir"]}{types.lower()}_flag' - + f"{self._img_number}.fits" - ) - - if self._config[types]["reg_file"] is None: - reg = default_reg - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute(f"rm {reg}") - - - else: - reg = self._config[types]["reg_file"] - - if not file_io.BaseCatalogue(reg)._file_exists(reg): - raise file_io.BaseCatalogue.CatalogFileNotFound(reg) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg} " - + f'-POLY_OUTFLAGS {self._config[types]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - - elif types == "ALL": - - default_reg = [ - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo{self._img_number}.reg" - ), - ( - f'{self._config["PATH"]["temp_dir"]}' - + f"spike{self._img_number}.reg" - ), - ] - default_out = ( - f'{self._config["PATH"]["temp_dir"]}' - + f"halo_spike_flag{self._img_number}.fits" - ) - - if self._config["HALO"]["reg_file"] is None: - reg = default_reg - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - self._rm_reg_stdout, self._rm_reg_stderr = execute( - f"rm {reg[0]} {reg[1]}" - ) - else: - reg = [ - self._config["HALO"]["reg_file"], - self._config["SPIKE"]["reg_file"], - ] - - for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) - - cmd = ( - f'{self._config["PATH"]["WW"]} ' - + f'-c {self._config["PATH"]["WW_configfile"]} ' - + f"-WEIGHT_NAMES {self._weight_fullpath} " - + f"-POLY_NAMES {reg[0]},{reg[1]} " - + f'-POLY_OUTFLAGS {self._config["HALO"]["flag"]},' - + f'{self._config["SPIKE"]["flag"]} ' - + f'-FLAG_NAMES "" -OUTFLAG_NAME {default_out} ' - + '-OUTWEIGHT_NAME ""' - ) - - self._WW_stdout, self._WW_stderr = execute(cmd) - - else: - raise ValueError("Types must be in ['HALO','SPIKE','ALL']") - - if (self._WW_stderr != "") or (self._rm_reg_stderr != ""): - self._err = True - - def _build_final_mask( - self, - path_mask1, - path_mask2=None, - masks_internal=None, - path_external_flag=None, - ): - """Create Final Mask. - - Create the final mask by combining the individual masks. - - Parameters - ---------- - path_mask1 : str - Path to a mask (FITS format) - path_mask2 : str, optional - Path to a mask (FITS format) - masks_internal : dict, optional - Internally created masks - path_external_flag : str, optional - Path to an external flag file - - Returns - ------- - numpy.ndarray - Array containing the final mask - - Raises - ------ - ValueError - If all masks are of type ``None`` - TypeError - If border is not a Numpy array - TypeError - If Messier mask is not a Numpy array - - """ - final_mask = None - - if path_mask1 is None and path_mask2 is None and not masks_internal: - raise ValueError( - "No paths to mask files containing halos and/or spikes," - + " borders, or deep-sky objects provided" - ) - - if path_mask1 is not None: - mask1 = file_io.FITSCatalogue(path_mask1, hdu_no=self._hdu) - mask1.open() - dat = mask1.get_data() - final_mask = dat[:, :] - - if path_mask2 is not None: - mask2 = file_io.FITSCatalogue(path_mask2, hdu_no=self._hdu) - mask2.open() - if final_mask is not None: - final_mask += mask2.get_data()[:, :] - else: - final_mask = mask2.get_data()[:, :] - - for typ in masks_internal: - if masks_internal[typ] is not None: - if type(masks_internal[typ]) is np.ndarray: - if final_mask is not None: - final_mask += masks_internal[typ] - else: - final_mask = masks_internal[typ] - else: - raise TypeError( - f"internally created mask of type {typ} " - + "has to be numpy.ndarray" - ) - - if path_external_flag is not None: - external_flag = file_io.FITSCatalogue( - path_external_flag, - hdu_no=self._hdu, - ) - external_flag.open() - if final_mask is not None: - final_mask = final_mask.astype(np.int16, copy=False) - try: - ext_flag = external_flag.get_data() - except: - self._w_log.info( - "Problem while getting external flag data. Check" - + f" whether file {path_external_flag} is not corrupt" - ) - raise - final_mask += ext_flag[:, :] - else: - final_mask = external_flag.get_data()[:, :] - external_flag.close() - - return final_mask.astype(np.int16, copy=False) - - def _mask_to_file(self, input_mask, output_fullpath): - """Mask to File. - - Save the mask to a fits file. - - Parameters - ---------- - input_mask : numpy.ndarray - Mask to save - output_fullpath : str - Path of the output file - - Raises - ------ - ValueError - If input_mask is type ``None`` - ValueError - If output_fullpath is type ``None`` - - """ - if input_mask is None: - raise ValueError("input mask file path not provided") - if output_fullpath is None: - raise ValueError("output mask file path not provided") - - out = file_io.FITSCatalogue( - output_fullpath, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - hdu_no=0, - ) - out.save_as_fits(data=input_mask, image=True) - - if self._config["MD"]["make"]: - out.open() - out.add_header_card( - "MRATIO", - self._ratio, - "ratio missing_pixels/all_pixels", - ) - out.add_header_card( - "MFLAG", - self._config["MD"]["im_flagged"], - f'threshold value {self._config["MD"]["thresh_flag"]:.3}', - ) - - # Write WCS information to header - if self._wcs: - header_wcs = self._wcs.to_header() - for card in header_wcs: - out.add_header_card( - card, - header_wcs[card], - header_wcs.comments[card], - ) - out.close() - - def _get_temp_dir_path(self, temp_dir_path): - """Get Temporary Directory Path. - - Create the path and the directory for temporary files. - - Parameters - ---------- - temp_dir_path : str - Path to the temporary directory, a value of ``OUTPUT`` will include - the temporary files in the run directory - - Returns - ------- - str - Path to the temporary directory - - Raises - ------ - ValueError - If ``temp_dir_path`` is of type ``None`` - - """ - if temp_dir_path is None: - raise ValueError("Temporary directory path not provided") - - path = temp_dir_path.replace(" ", "") - - if path == "OUTPUT": - path = f"{self._output_dir}/temp" - - path += "/" - if not os.path.isdir(path): - mkdir(path) - - return path diff --git a/src/shapepipe/modules/mask_query_package/__init__.py b/src/shapepipe/modules/mask_query_package/__init__.py new file mode 100644 index 000000000..fed9dcd92 --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/__init__.py @@ -0,0 +1,103 @@ +"""MASK QUERY PACKAGE. + +This package contains the module for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +:Parent module: ``sextractor_runner`` + +:Input: Single-exposure single-CCD SExtractor catalogue + +:Output: The same catalogue with an added ``MASK_EXT`` column + +Two classes of mask +=================== + +ShapePipe distinguishes them, and they are not interchangeable: + +**Instrument flags** mark a CORRUPTED MEASUREMENT — bad columns, saturation — +in the flag image delivered with each exposure. A flagged pixel carries no +usable signal, so these are the only masks that reject anything inside the +pipeline: ``setools`` drops flagged stars via ``IMAFLAGS_ISO``, and ``ngmix`` +zero-weights flagged pixels and drops epochs that lose too many. + +**Healsparse masks** are sky-fixed LOCATION flags — a star halo, a manual +region, a band with no data. They say where an object is, not that its pixels +are broken, so what to do about one is an analysis decision. They are queried +into columns and every rejection happens downstream: ``MASK_EXT`` here, +``MASK_`` in ``make_cat``. Nothing in the pipeline cuts on either. + +Description +=========== + +This module sits between SExtractor and ``setools`` on the exposure chain: it +reads each detection's windowed world position (``XWIN_WORLD``, ``YWIN_WORLD`` +in the ``LDAC_OBJECTS`` extension) and looks it up in the configured healsparse +maps, writing one integer column: + +``MASK_EXT`` + ``0`` for an object no configured map flags, nonzero otherwise. What the + nonzero value *is* depends on the maps: a boolean map — which is what the + UNIONS per-bit products are, and what the shipped config names — can only + contribute ``1``, so with those the column is 0/1 and says nothing about + which map fired. An integer map contributes its own value (optionally + ``& MASK_BITS``), and contributions are OR-ed, so a bit-packed map does + carry its bits through. Nothing downstream reads more than ``== 0``. + +The single column exists because ``setools`` expressions support only +``< > <= >= == !=`` — no bitwise operators — so any bit selection has to happen +here, leaving the config a plain test for zero. + +Off by default, and a no-op when off +==================================== + +``MASK_PATHS`` ships COMMENTED OUT. With no maps configured the module is a +strict no-op: the catalogue is copied through unchanged, with no ``MASK_EXT`` +column at all — the same gating ``make_cat`` gives ``MASK_EXT_PATHS``. It stays +in the ``MODULE`` chain either way, so enabling the query is uncommenting one +line and never editing the chain. + +Even with maps configured, nothing cuts on the result. The PSF star selection +deliberately starts from outlier rejection alone, rejecting only on the +instrument flags, and ``MASK_EXT`` is the configurable pickup if that proves +insufficient. Writing the column without cutting on it is what lets the effect +of a mask on the star sample be *measured* before it is imposed. + +That pickup is one line: add ``MASK_EXT == 0`` beside each +``IMAFLAGS_ISO == 0`` in ``star_selection.setools``. The escape hatch is +deliberate, and that file's header says so. + +The lookup itself lives in :mod:`shapepipe.utilities.mask_query`, shared with +``make_cat``'s per-band ``MASK_`` columns, so the healsparse primitive is +written once. That module's docstring documents the off-coverage convention. + +What gets queried is deliberately narrow +======================================== + +``MASK_PATHS`` is a *list of maps to record against each PSF-star candidate*, +not a list of every mask that exists. The committed configs name exactly one +map — the UNIONS star-body product (bit 2). Halo bits 0 and 1 are excluded on +purpose: halos flag objects for the final catalogue, they say nothing about +whether a star is a good PSF sample (mask-force telecon, 2026-07-21). MaxiMask +is not queried here either. + +Widening it costs a config edit and no code — add a path. That is why the +contract is a path list rather than a bit mask: the UNIONS products are one +boolean map per bit, so choosing bits *is* choosing files, and ``MASK_BITS`` +exists only for integer maps that pack several bits into one file. + +Module-specific config file entries +=================================== + +MASK_PATHS : str + Comma-separated healsparse map paths to query +MASK_BITS : int, optional + Bit mask applied to integer maps (``value & MASK_BITS`` flags); default is + to flag on any nonzero value. Ignored for boolean maps, which flag on + ``True`` +PREFIX : str, optional + Output file prefix + +""" + +__all__ = ["mask_query"] diff --git a/src/shapepipe/modules/mask_query_package/mask_query.py b/src/shapepipe/modules/mask_query_package/mask_query.py new file mode 100644 index 000000000..aff212b5a --- /dev/null +++ b/src/shapepipe/modules/mask_query_package/mask_query.py @@ -0,0 +1,133 @@ +"""MASK QUERY. + +Class to flag SExtractor detections against external healsparse masks. + +:Author: Claude Fable 5, for PR #847 + +""" + +import shutil + +import numpy as np + +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + + +class MaskQuery(object): + """Mask Query. + + Query external healsparse masks at every detection of a SExtractor + catalogue and write the result as a single ``MASK_EXT`` column into a copy + of that catalogue. + + With no maps configured this is a strict no-op: the input catalogue is + copied through unchanged, with no ``MASK_EXT`` column, matching + ``make_cat``'s ``MASK_EXT_PATHS`` contract (absent key, nothing happens). + The copy is what keeps the module in the chain — ``setools`` reads this + module's output, so producing no file would break the chain rather than + disable the query. + + Parameters + ---------- + sexcat_path : str + Path to the input SExtractor catalogue + output_path : str + Path to the output catalogue + mask_paths : list + Paths to the healsparse maps to query; empty means no-op + bits : int, optional + Bit mask applied to integer maps; default ``None`` flags any nonzero + value + w_log : logging.Logger, optional + Logging instance + + """ + + def __init__( + self, + sexcat_path, + output_path, + mask_paths, + bits=None, + w_log=None, + ): + + self._sexcat_path = sexcat_path + self._output_path = output_path + self._mask_paths = mask_paths + self._bits = bits + self._w_log = w_log + + def process(self): + """Process. + + Query the masks and write the flagged catalogue. + + Returns + ------- + int + Number of flagged objects + + """ + if not self._mask_paths: + # No maps configured: copy the catalogue through byte-for-byte. + # A rewrite through FITSCatalogue would round-trip the LDAC HDUs + # for no reason; this way an unconfigured run is provably + # identical to its input. + shutil.copyfile(self._sexcat_path, self._output_path) + if self._w_log is not None: + self._w_log.info( + "No MASK_PATHS configured; passing " + + f"{self._sexcat_path} through unchanged, no MASK_EXT" + + " column written" + ) + return 0 + + ori_cat = file_io.FITSCatalogue( + self._sexcat_path, + SEx_catalogue=True, + ) + ori_cat.open() + data = ori_cat.get_data() + + # A CCD SExtractor found nothing on is tolerated all along this chain + # (setools' ~0.2% attrition, psfex_interp's floor=0 warn), so it must + # not be an error here either. An empty LDAC table has no columns to + # index, so read the positions only when there are rows, and still + # publish an output file — a missing sexcat_ext would look to the file + # handler like a crash rather than like an empty CCD. + if len(data) == 0: + ra = np.zeros(0) + dec = np.zeros(0) + if self._w_log is not None: + self._w_log.info( + "No detections in " + + f"{self._sexcat_path}; writing an empty MASK_EXT column" + ) + else: + ra = np.copy(data["XWIN_WORLD"]) + dec = np.copy(data["YWIN_WORLD"]) + + flag = mask_query_util.flag_positions( + self._mask_paths, + ra, + dec, + bits=self._bits, + w_log=self._w_log, + ) + # int32 is what the catalogue carries; the UNIONS bit table needs 12 + # bits, and no OR of it can overflow. + flag = flag.astype(np.int32) + + new_cat = file_io.FITSCatalogue( + self._output_path, + SEx_catalogue=True, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + ori_cat.add_col( + "MASK_EXT", flag, new_cat=True, new_cat_inst=new_cat + ) + ori_cat.close() + + return int(np.count_nonzero(flag)) diff --git a/src/shapepipe/modules/mask_query_runner.py b/src/shapepipe/modules/mask_query_runner.py new file mode 100644 index 000000000..a636560b2 --- /dev/null +++ b/src/shapepipe/modules/mask_query_runner.py @@ -0,0 +1,76 @@ +"""MASK_QUERY RUNNER. + +Module runner for ``mask_query``. + +:Author: Claude Fable 5, for PR #847 + +""" + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.modules.module_decorator import module_runner +from shapepipe.utilities import mask_query as mask_query_util + + +@module_runner( + version="1.0", + input_module="sextractor_runner", + file_pattern=["sexcat"], + file_ext=[".fits"], + depends=["numpy", "healsparse"], +) +def mask_query_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Mask Query Runner.""" + sexcat_path = input_file_list[0] + + # Get file prefix (optional) + if config.has_option(module_config_sec, "PREFIX"): + prefix = config.get(module_config_sec, "PREFIX") + if (prefix.lower() != "none") & (prefix != ""): + prefix = prefix + "_" + else: + prefix = "" + else: + prefix = "" + + # Absent (or empty) MASK_PATHS is a no-op, not an error — the same gating + # make_cat gives MASK_EXT_PATHS. The module stays in the MODULE chain and + # passes the catalogue through, so enabling the query is a config edit and + # disabling it never means editing the chain. + if config.has_option(module_config_sec, "MASK_PATHS"): + mask_paths = mask_query_util.parse_map_paths( + config.getexpanded(module_config_sec, "MASK_PATHS") + ) + else: + mask_paths = [] + + # Any nonzero map value flags unless a bit selection is given + if config.has_option(module_config_sec, "MASK_BITS"): + bits = config.getint(module_config_sec, "MASK_BITS") + else: + bits = None + + output_path = ( + f'{run_dirs["output"]}/{prefix}sexcat_ext{file_number_string}.fits' + ) + + mq_inst = MaskQuery( + sexcat_path, + output_path, + mask_paths, + bits=bits, + w_log=w_log, + ) + n_flagged = mq_inst.process() + + if mask_paths: + w_log.info(f"MASK_EXT nonzero for {n_flagged} objects") + + # No return objects + return None, None diff --git a/src/shapepipe/modules/mask_runner.py b/src/shapepipe/modules/mask_runner.py deleted file mode 100644 index 13fbf9b6c..000000000 --- a/src/shapepipe/modules/mask_runner.py +++ /dev/null @@ -1,120 +0,0 @@ -"""MASK RUNNER. - -Module runner for ``mask``. - -:Author: Axel Guinot, Martin Kilbinger - -""" - -from shapepipe.modules.mask_package.mask import Mask -from shapepipe.modules.module_decorator import module_runner - - -@module_runner( - version="1.0", - file_pattern=["image", "weight", "flag"], - file_ext=[".fits", ".fits", ".fits"], - depends=["numpy", "astropy"], - executes=["weightwatcher"], - numbering_scheme="_0", -) -def mask_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Mask Runner.""" - # Get number of input files - n_inputs = len(input_file_list) - - # Set options for 2 inputs - if n_inputs == 2: - ext_flag_name = None - ext_star_cat = None - - # Set options for 3 inputs - elif n_inputs == 3: - if config.getboolean(module_config_sec, "USE_EXT_FLAG"): - ext_flag_name = input_file_list[2] - ext_star_cat = None - elif config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = None - ext_star_cat = input_file_list[2] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag or " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Set options for 4 inputs - elif n_inputs == 4: - if config.getboolean( - module_config_sec, "USE_EXT_FLAG" - ) and config.getboolean(module_config_sec, "USE_EXT_STAR"): - ext_flag_name = input_file_list[2] - ext_star_cat = input_file_list[3] - else: - raise ValueError( - f"Found {n_inputs} inputs but was expecting external flag and " - + "external star catalogue in the MASK_RUNNER section of the " - + "config file." - ) - - # Raise error for invalid settings - else: - raise ValueError( - f'Found {n_inputs} inputs and these must be "image", "weight" and ' - + '"ext_flags", "ext_star_cat" (optional). Check the MASK_RUNNER ' - + "section of the config file to make sure you have the " - + "appropriate settings." - ) - - # Get path to mask configuration options - config_file = config.getexpanded(module_config_sec, "MASK_CONFIG_PATH") - - # Get mask HDU number - if config.has_option(module_config_sec, "HDU"): - hdu = config.getint(module_config_sec, "HDU") - else: - hdu = 0 - - # Get mask mask file name prefix - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - else: - prefix = "" - - outname_base = "flag" - - # Path to check for already created mask files - if config.has_option(module_config_sec, "CHECK_EXISTING_DIR"): - check_existing_dir = config.getexpanded( - module_config_sec, "CHECK_EXISTING_DIR" - ) - else: - check_existing_dir = None - - # Create instance of Mask - mask_inst = Mask( - *input_file_list[:2], - image_prefix=prefix.replace(" ", ""), - image_num=file_number_string, - config_filepath=config_file, - output_dir=run_dirs["output"], - path_external_flag=ext_flag_name, - outname_base=outname_base, - star_cat_path=ext_star_cat, - check_existing_dir=check_existing_dir, - hdu=hdu, - w_log=w_log, - ) - - # Process module - stdout, stderr = mask_inst.make_mask() - - # Return stdout and stderr - return stdout, stderr diff --git a/src/shapepipe/modules/random_cat_package/__init__.py b/src/shapepipe/modules/random_cat_package/__init__.py deleted file mode 100644 index 32c6f64b2..000000000 --- a/src/shapepipe/modules/random_cat_package/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""RANDOM CATALOGUE PACKAGE. - -This package contains the module for ``random_cat``. - -:Author: Martin Kilbinger - -:Parent module: None - -:Input: Images and masks - -:Output: Random catalogue FITS file - -Description -=========== - -This module creates a random catalogue, and computes the tile area accounting -for overlapping and masked regions. - -Module-specific config file entries -=================================== - -N_RANDOM : float - The number of random objects requested on output -DENSITY : bool, optional - Option to interpret the number of random objects per square degree; the - default is ``False`` -SAVE_MASK_AS_HEALPIX : bool - Output healpix mask if ``True`` -HEALPIX_OUT_FILE_BASE : str, optional - Output halpix mask file base name; used only if SAVE_MASK_AS_HEALPIX is - ``True`` -HEALPIX_OUT_NSIDE : int, optional - Output healpix mask nside; used only if SAVE_MASK_AS_HEALPIX is ``True`` - -""" - -__all__ = ["random_cat.py"] diff --git a/src/shapepipe/modules/random_cat_package/random_cat.py b/src/shapepipe/modules/random_cat_package/random_cat.py deleted file mode 100644 index cabbd18e0..000000000 --- a/src/shapepipe/modules/random_cat_package/random_cat.py +++ /dev/null @@ -1,235 +0,0 @@ -"""RANDOM CATALOGUE. - -This module contains a class to create a random catalogue, and to compute -the tile area accounting for overlapping and masked regions. - -:Author: Martin Kilbinger - -""" - -import os -import re - -import numpy as np - -import astropy.io.fits as fits -from astropy import wcs -from astropy.table import Table - -from reproject import reproject_to_healpix - -from shapepipe.pipeline import file_io -from shapepipe.utilities import cfis - - -class RandomCat: - """Random Catalogue. - - This class creates a random catalogue given a mask FITS file. - - Parameters - ---------- - input_image_path : str - Path to input image file - input_mask_path : str - Path to input mask file - output_dir : str - Output directory - file_number_pattern : str - ShapePipe image ID string - output_file_pattern : str - Output file pattern (base name) for random catalogue - n_rand : float - Number of random objects on output - density : bool - ``n_rand`` is interpreted per square degrees if ``True`` - w_log : logging.Logger - Logging instance - healpix_options : dict - Parameters for HEALPix output mask file - """ - - def __init__( - self, - input_image_path, - input_mask_path, - output_dir, - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ): - - self._input_image_path = input_image_path - self._input_mask_path = input_mask_path - self._output_dir = output_dir - self._file_number_string = file_number_string - self._output_file_pattern = output_file_pattern - self._n_rand = n_rand - self._density = density - self._w_log = w_log - self._healpix_options = healpix_options - - def save_as_healpix(self, hdu_mask, header): - """Save As Healpix. - - Save mask as healpix FITS file. - - Parameters - ---------- - hdu_mask : class HDUList - HDU with 2D pixel mask image - header : class Header - Image header with WCS information - - """ - if not self._healpix_options: - return - - mask_1d, footprint = reproject_to_healpix( - (hdu_mask, header), - 'galactic', - nside=self._healpix_options['OUT_NSIDE'] - ) - - t = Table() - t['flux'] = mask_1d - t.meta['ORDERING'] = 'RING' - t.meta['COORDSYS'] = 'G' - t.meta['NSIDE'] = self._healpix_options['OUT_NSIDE'] - t.meta['INDXSCHM'] = 'IMPLICIT' - - output_path = ( - f'{output_dir}/{self._healpix_options["FILE_BASE"]}-' - + f'{file_number_string}.fits' - ) - t.write(output_path) - - def process(self): - """Process. - - Main function to identify exposures. - - """ - # Read image FITS file header - try: - img = fits.open(self._input_image_path) - header = img[0].header - except (OSError, IOError) as error: - # FITS file might contain only header. - # Try as ascii file - try: - fin = open(self._input_image_path) - header = fits.Header.fromtextfile(fin) - fin.close() - except Exception: - raise - - # Get WCS - WCS = wcs.WCS(header) - - # Read mask FITS file - hdu_mask = fits.open(self._input_mask_path) - mask = hdu_mask[0].data - - # Save mask in healpix format (if option is set) - self._save_as_healpix(hdu_mask, header) - - # Number of pixels - n_pix_x = mask.data.shape[0] - n_pix_y = mask.data.shape[1] - n_pix = n_pix_x * n_pix_y - - # Number of non-masked pixels - n_unmasked = len(np.where(mask == 0)[0]) - - # Compute various areas - - # Pixel area in deg^2 - area_pix = wcs.utils.proj_plane_pixel_area(WCS) - - # Tile area - area_deg2 = area_pix * n_pix - - # Area of unmasked region - area_deg2_eff = area_pix * n_unmasked - - # Compute number of requested objects - if n_unmasked > 0: - if not self._density: - # Use value from config file - n_obj = self._n_rand - else: - # Compute number of objects from density - n_obj = int( - self._n_rand / area_deg2 * area_deg2_eff / area_deg2 - ) - - # Check that a reasonably large number of pixels is not masked - if n_unmasked < n_obj: - raise ValueError( - f"Number of un-masked pixels {n_unmasked} is smaller " - + f"than number of random objects requested {n_obj}" - ) - - else: - n_obj = 0 - - self._w_log.info(f"Creating {n_obj} random objects") - - # Draw points until n are in mask - n_found = 0 - xy_rand = [] - while n_found < n_obj: - idx_x = np.random.randint(n_pix_x) - idx_y = np.random.randint(n_pix_y) - - # Add points with additional random sub-pixel value - if mask[idx_x, idx_y] == 0: - d = np.random.random(2) - # MKDEBUG: the following seems to work, x and y interchanged - xy_rand.append([idx_y + d[1], idx_x + d[0]]) - n_found = n_found + 1 - xy_rand = np.array(xy_rand) - - # Transform to WCS - res = WCS.all_pix2world(xy_rand, 1) - if n_unmasked > 0: - ra_rand = res[:, 0] - dec_rand = res[:, 1] - x_rand = xy_rand[:, 0] - y_rand = xy_rand[:, 1] - else: - ra_rand = [] - dec_rand = [] - x_rand = [] - y_rand = [] - - # Tile ID - output_path = ( - f"{self._output_dir}/{self._output_file_pattern}-" - + f"{self._file_number_string}.fits" - ) - file_base = os.path.splitext(file_name)[0] - tile_ID_str = re.split("-", file_base)[1:] - tile_id = float(".".join(tile_ID_str)) - tile_id_array = np.ones(n_obj) * tile_id - - # Write to output - cat_out = [ra_rand, dec_rand, x_rand, y_rand, tile_id_array] - column_names = ["RA", "DEC", "x", "y", "TILE_ID"] - - # TODO: Add units to header - output = file_io.FITSCatalogue( - output_path, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite - ) - output.save_as_fits(cat_out, names=column_names) - - # Write area information to log file - self._w_log.info(f"Total area = {area_deg2:.4f} deg^2") - self._w_log.info(f"Unmasked area = {area_deg2_eff:.4f} deg^2") - self._w_log.info( - f"Ratio masked to total pixels = {n_unmasked / n_pix:.3f}" - ) diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py deleted file mode 100644 index c85854cb5..000000000 --- a/src/shapepipe/modules/random_cat_runner.py +++ /dev/null @@ -1,79 +0,0 @@ -"""RANDOM CAT RUNNER. - -Module runner for ``random_cat``. - -:Author: Martin Kilbinger - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.random_cat_package.random_cat import RandomCat - - -@module_runner( - version="1.1", - file_pattern=["image", "pipeline_flag"], - file_ext=[".fits", "fits"], - depends=["astropy"], - numbering_scheme="_0", -) -def random_cat_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Random Catalogue Runner.""" - # Get input file names of image and mask - input_image_name = input_file_list[0] - input_mask_name = input_file_list[1] - - # Set output file name - if config.has_option(module_config_sec, "OUTPUT_FILE_PATTERN"): - output_file_pattern = config.get( - module_config_sec, "OUTPUT_FILE_PATTERN" - ) - else: - output_file_pattern = "random_cat" - - # Get number of random objects requested on output - n_rand = config.getfloat(module_config_sec, "N_RANDOM") - - # Flag whether n_rand is total (DENSITY=False, default) - # or per square degree (DENSITY=True) - if config.has_option(module_config_sec, "DENSITY"): - density = config.getboolean(module_config_sec, "DENSITY") - else: - density = False - - # Get healpix output options - save_mask_as_healpix = config.getboolean( - module_config_sec, "SAVE_MASK_AS_HEALPIX" - ) - if save_mask_as_healpix: - healpix_options = {} - for option_trunc in ['FILE_BASE', 'OUT_NSIDE']: - option = f'HEALPIX_OUT_{option_trunc}' - healpix_options[option_trunc] = config.get( - module_config_sec, option - ) - # Create rand cat class instance - rand_cat_inst = RandomCat( - input_image_name, - input_mask_name, - run_dirs["output"], - file_number_string, - output_file_pattern, - n_rand, - density, - w_log, - healpix_options, - ) - - # Run processing - rand_cat_inst.process() - - # No return objects - return None, None diff --git a/src/shapepipe/modules/sextractor_package/__init__.py b/src/shapepipe/modules/sextractor_package/__init__.py index abb0b4572..40f40ca91 100644 --- a/src/shapepipe/modules/sextractor_package/__init__.py +++ b/src/shapepipe/modules/sextractor_package/__init__.py @@ -4,10 +4,10 @@ :Author: Axel Guinot -:Parent modules: ``mask_runner``, ``merge_headers_runner`` (the latter only +:Parent modules: ``split_exp_runner``, ``merge_headers_runner`` (the latter only when ``MAKE_POST_PROCESS`` is ``True``) -:Input: Single-exposure single-CCD image, weight and flag files +:Input: Single-exposure single-CCD image, weight and instrument flag files :Output: SExtractor output catalogue diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index c36a8596d..e45885b06 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -19,7 +19,7 @@ # first three entries only. @module_runner( version="1.0.1", - input_module=["mask_runner", "merge_headers_runner"], + input_module=["split_exp_runner", "merge_headers_runner"], file_pattern=["image", "weight", "flag", "log_exp_headers"], file_ext=[".fits", ".fits", ".fits", ".sqlite"], executes=["source-extractor"], diff --git a/src/shapepipe/utilities/__init__.py b/src/shapepipe/utilities/__init__.py index 5f0eab2d6..42e234003 100644 --- a/src/shapepipe/utilities/__init__.py +++ b/src/shapepipe/utilities/__init__.py @@ -7,4 +7,4 @@ """ -__all__ = ["file_system", "cfis", "galaxy", "summary"] +__all__ = ["file_system", "cfis", "galaxy", "mask_query", "summary"] diff --git a/src/shapepipe/utilities/file_io.py b/src/shapepipe/utilities/file_io.py deleted file mode 100644 index 8a0964aa0..000000000 --- a/src/shapepipe/utilities/file_io.py +++ /dev/null @@ -1,45 +0,0 @@ -"""FILE I/O UTILITIES. - -Small, dependency-light helpers for publishing files the rest of the pipeline -treats as a cache. - -:Author: consolidated from workflow/scripts/star_cats.py and - scripts/python/create_star_cat.py - -""" - -import os - - -def write_atomic(table, path): - """Publish ``table`` at ``path`` all-or-nothing. - - Every caller's only cache test is existence (``Path.exists`` / - ``os.path.isfile``), so a write killed part-way -- job timeout, OOM, node - failure -- would otherwise leave a truncated FITS that every later run - trusts forever, and ``test -s`` passes on partial bytes. Writing to a temp - and renaming makes the visible file all-or-nothing: ``os.replace`` is atomic - within a directory. - - The temp keeps the target's suffix, because astropy picks its writer from - the extension. It is dot-prefixed and PID-tagged so it stays out of the - ``star_chunk-*`` / ``star_cat*`` globs the rules use, and two concurrent - writers cannot collide. - - Parameters - ---------- - table : astropy.table.Table - Table to write - path : str or pathlib.Path - Destination path - - """ - path = os.fspath(path) - directory, name = os.path.split(path) - tmp = os.path.join(directory or ".", f".tmp-{os.getpid()}-{name}") - try: - table.write(tmp, overwrite=True) - os.replace(tmp, path) - finally: - if os.path.exists(tmp): - os.remove(tmp) diff --git a/src/shapepipe/utilities/focal_plane.py b/src/shapepipe/utilities/focal_plane.py deleted file mode 100644 index 32770bd06..000000000 --- a/src/shapepipe/utilities/focal_plane.py +++ /dev/null @@ -1,92 +0,0 @@ -"""FOCAL PLANE GEOMETRY. - -The sky footprint of a MegaCam exposure, read from its image headers. Both -star-catalogue producers need exactly this and used to carry their own copy of -it -- ``workflow/scripts/star_cats.py`` (the HEALPix chunk store's ``cut``) and -``scripts/python/create_star_cat.py`` (the one-cone-per-exposure path the store -replaced). They must agree on which sky an exposure covers, so there is one -definition of it. - -:Author: consolidated from the two star-catalogue scripts - -""" - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.wcs import WCS - - -def get_wcs(header): - """Build the WCS by hand, from the linear terms only. - - Deliberately NOT ``WCS(header)``: it sidesteps distortion-convention - incompatibilities between these headers and astropy, and a footprint needs - nothing finer than the linear terms. - - Parameters - ---------- - header : astropy.io.fits.Header - Image header - - Returns - ------- - astropy.wcs.WCS - WCS object - - """ - final_wcs = WCS(naxis=2) - final_wcs.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] - try: - final_wcs.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] - except KeyError: - final_wcs.wcs.cunit = ["deg", "deg"] - final_wcs.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] - final_wcs.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] - final_wcs.wcs.cd = [ - [header["CD1_1"], header["CD1_2"]], - [header["CD2_1"], header["CD2_2"]], - ] - - return final_wcs - - -def ccd_center_and_radius(header): - """Return ``(ra_deg, dec_deg, radius_deg)`` for a single CCD. - - The radius is the half-diagonal: centre to the ``(0, 0)`` corner. - - """ - w = get_wcs(header) - (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( - [[header["NAXIS1"] / 2.0, header["NAXIS2"] / 2.0], [0, 0]], 1) - center = SkyCoord(ra_c * u.deg, dec_c * u.deg) - radius = center.separation(SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg - return float(ra_c), float(dec_c), float(radius) - - -def focal_plane_disc(image, n_ccd=40): - """Return ``(ra_deg, dec_deg, radius_deg)`` covering all CCDs of one exposure. - - The centre is the mean of the CCD centres; the radius is the largest - centre-to-CCD-centre distance plus that CCD's own half-diagonal. - - ONE ``fits.open`` for all the extensions: ``fits.getheader(image, ext)`` - opens the file, walks the HDU list to ``ext`` and closes again, so a loop - over it costs ``n_ccd`` opens and O(n^2) header seeks. - - """ - centers, radii = [], [] - with fits.open(image) as hdul: - for ext in range(1, n_ccd + 1): - ra_c, dec_c, radius = ccd_center_and_radius(hdul[ext].header) - centers.append((ra_c, dec_c)) - radii.append(radius) - - ras = np.array([c[0] for c in centers]) - decs = np.array([c[1] for c in centers]) - center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) - seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg - return (float(center.ra.deg), float(center.dec.deg), - float(np.max(seps + np.array(radii)))) diff --git a/src/shapepipe/utilities/mask_query.py b/src/shapepipe/utilities/mask_query.py new file mode 100644 index 000000000..982bfd3d3 --- /dev/null +++ b/src/shapepipe/utilities/mask_query.py @@ -0,0 +1,294 @@ +"""MASK QUERY. + +The one healsparse lookup in ShapePipe. + +ShapePipe does not generate or rasterize masks. Sky-fixed masks are supplied +as healsparse maps and are consumed by *querying them at object positions*: +every object gets its mask value(s) as catalogue columns, and rejection happens +at the catalogue level, never at the pixel level. The only mask that still +reaches pixels is the per-exposure instrument flag image delivered with it. + +Two callers share the primitive defined here: + +* ``make_cat`` writes one ``MASK_`` column per configured map, carrying + the map value verbatim (no interpretation, no filtering); +* ``mask_query`` writes a single integer ``MASK_EXT`` column onto the exposure + SExtractor catalogue, combining the configured maps into "clean (0) or + flagged (nonzero)" so that ``setools`` — whose expression language has no + bitwise operators — *could* cut on ``MASK_EXT == 0``. The shipped selection + does not; the column is carried for measurement (see that module). + +Partial reads +------------- +Never read a whole map. The UNIONS products are ~550 MB each and ``mask_query`` +runs PER CCD, so a full read would cost ~22 GB of I/O per 40-CCD exposure to +answer questions about a 0.06 deg² footprint. Instead the coverage index is +read first (``HealSparseCoverage``, a few hundred kB), the coverage pixels the +catalogue actually touches are computed with ``hpgeom.angle_to_pixel`` at the +map's ``nside_coverage``, and only those get loaded. + +Every queried position falls inside exactly one coverage pixel and all of them +are requested, so the padding by ``hpgeom.neighbors`` is insurance rather +than necessity — at ``nside_coverage=128`` a coverage pixel of the star map +is ~23 kB, so the padding costs under a megabyte and buys immunity to any +edge convention we did not think of. ``test_partial_read_matches_full`` is +what actually holds the two paths equal. + +Measured on the DR6 star-body map — the bit-2 rung of the ladder the configs +name ``mask_ugriz_nside131072_n4.hsp``, run against the staged single-band copy +``mask_r_nside131072_n4.hsp``, 583 MB, ``nside_coverage=128`` — for 2000 +positions in one CCD-sized box, one process each: partial 0.102 s / 157 MiB +peak RSS, full 12.8 s / 3364 MiB, identical values. Per 40-CCD exposure that +is ~4 s against ~8.5 min of map reading, and the query adds ~0.15 GB to a rule +already asking for 16 GB. + +Coverage +-------- +``healsparse.HealSparseMap.get_values_pos`` returns a map's *sentinel* for +positions outside its coverage: ``False`` for boolean maps, and typically +``-1`` for integer maps. ``make_cat`` passes that sentinel through verbatim, +which is the documented off-map flag for the final catalogue. + +Coverage is reported from the COVERAGE MASK, not ``valid_mask=True``. For a +boolean map — which is what the UNIONS per-bit products are — healsparse +stores only the ``True`` pixels, so ``valid_mask`` returns the value itself +and cannot tell "inside the footprint and clean" from "outside it entirely". +The coverage mask can, at ``nside_coverage`` resolution, which is the scale the +question is asked at anyway: does this map reach this exposure at all? + +``flag_positions`` treats off-coverage as **not flagged**, for both map kinds. +This makes the integer case agree with the boolean case (whose sentinel is +literally ``False``) rather than diverge from it, and it keeps a map whose +coverage does not reach an exposure from silently rejecting every star on it. +That is also why the all-off-coverage case is logged as a WARNING: it is +indistinguishable from "nothing is masked here" in the output column, so it has +to be distinguishable in the log. + +:Author: Claude Fable 5, for PR #847 + +""" + +import numpy as np + + +def parse_map_paths(paths_str): + """Parse Map Paths. + + Parse a comma-separated list of healsparse map paths. + + Parameters + ---------- + paths_str : str + Comma-separated map paths, e.g. ``/a/star.hsp, /b/maximask.hsp`` + + Returns + ------- + list + Map paths, stripped of surrounding whitespace, empty entries dropped + + """ + return [path.strip() for path in paths_str.split(",") if path.strip()] + + +def _covering_pixels(coverage, ra, dec): + """Covering Pixels. + + The map's coverage pixels touched by these positions, padded by their + neighbours and intersected with what the map actually holds. + + Parameters + ---------- + coverage : healsparse.HealSparseCoverage + Coverage index of the map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Coverage pixel indices to load, possibly empty + + """ + import hpgeom + + nside_coverage = coverage.nside_coverage + touched = np.unique( + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ) + padded = np.unique( + np.concatenate( + [touched, hpgeom.neighbors(nside_coverage, touched).ravel()] + ) + ) + # neighbors() returns -1 for a non-existent neighbour. + padded = padded[padded >= 0] + + return padded[coverage.coverage_mask[padded]] + + +def query_map_coverage(path, ra, dec): + """Query Map With Coverage. + + Read only the part of a healsparse map these positions need, and return + both its value at each position and whether each position is inside the + map's coverage. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + tuple + ``(values, in_coverage)`` — the map value at each position (the map's + sentinel outside coverage) and a boolean array, both of length + ``len(ra)`` + + """ + import healsparse + import hpgeom + + ra = np.asarray(ra) + dec = np.asarray(dec) + + coverage = healsparse.HealSparseCoverage.read(path) + nside_coverage = coverage.nside_coverage + + in_coverage = coverage.coverage_mask[ + hpgeom.angle_to_pixel(nside_coverage, ra, dec, nest=True) + ] + + pixels = _covering_pixels(coverage, ra, dec) + + if pixels.size == 0: + # healsparse raises when no requested pixel is in the coverage map, so + # the empty case is answered without asking it: load one arbitrary + # coverage pixel purely to learn the dtype and sentinel, and return + # that sentinel everywhere. Same answer, one small read. + covered = np.flatnonzero(coverage.coverage_mask) + if covered.size == 0: + raise ValueError(f"healsparse map {path} has empty coverage") + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(covered[0])] + ) + values = np.full(ra.size, mask_map.sentinel, dtype=mask_map.dtype) + return values, in_coverage + + mask_map = healsparse.HealSparseMap.read( + path, pixels=[int(pixel) for pixel in pixels] + ) + values = np.asarray(mask_map.get_values_pos(ra, dec, lonlat=True)) + + return values, in_coverage + + +def query_map(path, ra, dec): + """Query Map. + + Return a healsparse map's value at each world position, reading only the + coverage pixels those positions touch. + + Parameters + ---------- + path : str + Path to the healsparse map + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + + Returns + ------- + numpy.ndarray + Map value at each position; positions outside the map's coverage carry + the map's sentinel value + + """ + values, _ = query_map_coverage(path, ra, dec) + + return values + + +def flag_positions(paths, ra, dec, bits=None, w_log=None): + """Flag Positions. + + Combine one or more healsparse masks into a single per-object integer flag. + + Each map contributes at each position: + + * boolean map: ``1`` where the map is ``True``, ``0`` elsewhere; + * integer map: the map value, optionally restricted to ``bits`` + (``value & bits``); ``0`` where the value is zero or the position is + outside coverage. + + Contributions are combined with a bitwise OR, so the returned flag is zero + for a clean object and carries the union of the bits that fired otherwise. + + Parameters + ---------- + paths : list + Paths to the healsparse maps to query + ra : numpy.ndarray + Right ascension in degrees + dec : numpy.ndarray + Declination in degrees + bits : int, optional + Bit mask applied to integer maps; default ``None`` means any nonzero + value flags + w_log : logging.Logger, optional + Logging instance + + Returns + ------- + numpy.ndarray + Integer flag per object, ``0`` for a clean object + + """ + ra = np.asarray(ra) + dec = np.asarray(dec) + flag = np.zeros(ra.size, dtype=np.int64) + + if ra.size == 0: + return flag + + for path in paths: + values, in_coverage = query_map_coverage(path, ra, dec) + + if values.dtype == bool: + contribution = values.astype(np.int64) + else: + integer = values.astype(np.int64) + # Off-coverage: the sentinel, negative by healsparse convention. + # Zeroed rather than OR-ed in, see this module's docstring. + integer = np.where(integer < 0, 0, integer) + if bits is not None: + integer &= bits + contribution = integer + + flag |= contribution + + n_off = int(np.count_nonzero(~in_coverage)) + if w_log is not None: + w_log.info( + f"Mask query {path}: " + f"{int(np.count_nonzero(contribution))}/{ra.size} objects " + f"flagged, {n_off} outside coverage" + ) + if n_off == ra.size: + # Every object reads the sentinel, so the column is all-zero + # and looks exactly like "nothing is masked here". Say it. + w_log.warning( + f"Mask query {path}: NO object is inside this map's" + + " coverage — the resulting MASK_EXT contribution is" + + " zero everywhere because the map does not reach these" + + " positions, not because they are clean." + ) + + return flag diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py deleted file mode 100644 index 0bf077a78..000000000 --- a/src/shapepipe/utilities/vizier.py +++ /dev/null @@ -1,101 +0,0 @@ -"""VIZIER QUERY UTILITY. - -Consolidated Vizier query helper used by the mask module (to fetch the -reference star catalogue) and by ``scripts/python/create_star_cat.py``. -Retries over a list of mirror servers at progressively longer timeouts. - -:Author: Martin Kilbinger - -""" - -import random -import time - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astroquery.vizier import Vizier - - -VIZIER_SERVERS = [ - "vizier.cds.unistra.fr", - "vizier.cfa.harvard.edu", - "vizier.iucaa.in", -] - -VIZIER_TIMEOUTS = [10, 20, 40] - - -def query_vizier(ra, dec, radius_arcmin, cat_id): - """Query a Vizier catalogue with retries over timeouts and mirror servers. - - Parameters - ---------- - ra : float - Right ascension in degrees. - dec : float - Declination in degrees. - radius_arcmin : float - Cone-search radius in arcminutes. - cat_id : str - Vizier catalogue identifier (e.g. ``"I/305/out"`` for GSC 2.3). - - Returns - ------- - astropy.table.Table - First result table returned by Vizier. - - Raises - ------ - IndexError - If all server/timeout combinations return an empty result. - - """ - # Empirically, single-precision input positions can cause Vizier to return - # empty lists for some exposures; force double precision. - p = np.array([ra, dec], dtype="double") - coord = SkyCoord(ra=p[0] * u.deg, dec=p[1] * u.deg, frame="icrs") - - # Stagger concurrent queries to avoid hammering a single mirror. - time.sleep(random.uniform(0, 5)) - - for attempt, timeout in enumerate(VIZIER_TIMEOUTS): - for server in VIZIER_SERVERS: - v = Vizier( - row_limit=-1, timeout=timeout, vizier_server=server - ) - # cache=False: astroquery otherwise pickles every HTTP response into - # $HOME/.astropy/cache/astroquery/Vizier, ~2 MB per query. The - # workflow already caches the RESULT as a FITS catalogue on scratch - # and skips the query when it hits, so the pickle is pure duplicate — - # and at campaign scale (~25k exposures) it is ~50 GB against a - # 50 GB home quota. Home is for source and config, not for a second - # copy of the survey. - result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id, - cache=False, - ) - if len(result) > 0: - print( - f"Vizier query successful " - f"(server={server}, timeout={timeout}s)" - ) - return result[0] - print( - f"Vizier returned empty list at {coord}, " - f"{radius_arcmin:.2f} arcmin, " - f"server={server}, timeout={timeout}s" - ) - if attempt < len(VIZIER_TIMEOUTS) - 1: - wait = 10 * 2**attempt - print( - f"All servers failed, retrying in {wait}s " - f"(attempt {attempt + 1}/{len(VIZIER_TIMEOUTS)}, " - f"next timeout={VIZIER_TIMEOUTS[attempt + 1]}s)" - ) - time.sleep(wait) - - raise IndexError( - f"Vizier astroquery returned empty list at {coord}, " - f"radius={radius_arcmin} arcmin, catalog={cat_id}" - ) diff --git a/tests/module/test_make_cat_mask_ext.py b/tests/module/test_make_cat_mask_ext.py new file mode 100644 index 000000000..91d0f5154 --- /dev/null +++ b/tests/module/test_make_cat_mask_ext.py @@ -0,0 +1,129 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MAKE_CAT external-mask columns. + +Exercises the optional per-band external healsparse mask lookup added to +``make_cat`` (PR #847 §4, the ShapePipe end of UNIONS-WL/spherex#38). A small +synthetic ``final_cat`` FITS carrying known ``XWIN_WORLD`` / ``YWIN_WORLD`` +object positions is queried against synthetic healsparse maps of known value, +locking in: (1) the ``MASK_`` column name and per-object values, (2) the +off-map sentinel (``-1`` for integer maps) written verbatim for objects outside +coverage, (3) multi-band handling, and (4) that absent config leaves the +catalogue untouched. +""" + +import numpy as np +import numpy.testing as npt +import pytest + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.make_cat_package import make_cat +from shapepipe.pipeline import file_io + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Object world positions (RA, Dec in degrees). The last object sits far from +# the map coverage so it exercises the off-map sentinel path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +def _make_map(value, dtype=np.int16, sentinel=-1): + """Build a healsparse map covering the first three RA/Dec positions. + + All covered pixels carry ``value``; everything else reads the sentinel. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=sentinel + ) + smap.update_values_pos( + RA[:3], DEC[:3], np.full(3, value, dtype=dtype), lonlat=True + ) + return smap + + +def _write_final_cat(path): + """Write a synthetic final_cat FITS with a RESULTS ext of known positions.""" + data = np.empty( + len(RA), + dtype=[ + ("NUMBER", "i4"), + ("XWIN_WORLD", "f8"), + ("YWIN_WORLD", "f8"), + ], + ) + data["NUMBER"] = np.arange(len(RA)) + data["XWIN_WORLD"] = RA + data["YWIN_WORLD"] = DEC + + cat = file_io.FITSCatalogue( + str(path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + cat.save_as_fits(data, ext_name="RESULTS") + return cat + + +def test_parse_mask_ext_paths(): + """band:path pairs parse into a stripped mapping, whitespace-tolerant.""" + parsed = make_cat.parse_mask_ext_paths( + "u:/a/mask_u.hsp, g:/b/mask_g.hsp,r:/c/mask_r.hsp" + ) + assert parsed == { + "u": "/a/mask_u.hsp", + "g": "/b/mask_g.hsp", + "r": "/c/mask_r.hsp", + } + + +def test_mask_ext_columns(tmp_path): + """Per-band columns carry the map value on-map and the sentinel off-map.""" + u_map = _make_map(16) + g_map = _make_map(32) + u_path = tmp_path / "mask_u.hsp" + g_path = tmp_path / "mask_g.hsp" + u_map.write(str(u_path)) + g_map.write(str(g_path)) + + cat_path = tmp_path / "final_cat-000.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue( + str(cat_path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + make_cat.save_mask_ext_data( + cat, + {"u": str(u_path), "g": str(g_path)}, + _NullLogger(), + ) + + cat.open() + data = cat.get_data() + # On-map objects (first three) carry the map value; the off-map object + # (last) carries the map's -1 sentinel. + npt.assert_array_equal(data["MASK_u"], [16, 16, 16, -1]) + npt.assert_array_equal(data["MASK_g"], [32, 32, 32, -1]) + # Integer dtype preserved from the map. + assert np.issubdtype(data["MASK_u"].dtype, np.integer) + cat.close() + + +def test_mask_ext_absent_is_noop(tmp_path): + """Not calling the lookup leaves the catalogue columns unchanged.""" + cat_path = tmp_path / "final_cat-001.fits" + _write_final_cat(cat_path) + + cat = file_io.FITSCatalogue(str(cat_path)) + cat.open() + cols = set(cat.get_data().dtype.names) + cat.close() + + assert cols == {"NUMBER", "XWIN_WORLD", "YWIN_WORLD"} + assert not any(name.startswith("MASK_") for name in cols) diff --git a/tests/module/test_mask_query.py b/tests/module/test_mask_query.py new file mode 100644 index 000000000..f52d2e669 --- /dev/null +++ b/tests/module/test_mask_query.py @@ -0,0 +1,348 @@ +"""UNIT TESTS FOR MODULE PACKAGE: MASK_QUERY. + +Exercises the exposure-side half of the query-everything mask design (PR #847): +``mask_query`` reads each SExtractor detection's windowed world position out of +the ``LDAC_OBJECTS`` extension, looks it up in the configured healsparse maps, +and writes a single integer ``MASK_EXT`` column into a NEW catalogue beside the +input. + +What is locked in here: (1) boolean maps flag with ``1``, (2) integer maps +contribute their value and ``MASK_BITS`` restricts which bits do, (3) the +off-coverage sentinel (``-1``) never flags — the one place this differs from +``make_cat``'s verbatim pass-through, argued in +:mod:`shapepipe.utilities.mask_query` — (4) several maps OR together, and +(5) the input catalogue is left untouched while the LDAC structure survives. +""" + +import pathlib + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits + +healsparse = pytest.importorskip("healsparse") + +from shapepipe.modules.mask_query_package.mask_query import MaskQuery +from shapepipe.pipeline import file_io +from shapepipe.utilities import mask_query as mask_query_util + +NSIDE_COVERAGE = 32 +NSIDE_SPARSE = 4096 + +# Detection world positions (RA, Dec in degrees). The last one sits far from +# every map's coverage, so it exercises the off-coverage path. +RA = np.array([10.0, 10.1, 10.2, 200.0]) +DEC = np.array([20.0, 20.1, 20.2, -40.0]) + + +class _NullLogger: + """Captures what the module logs, so coverage warnings can be asserted.""" + + def __init__(self): + self.info_msgs = [] + self.warning_msgs = [] + + def info(self, msg, *_a, **_kw): + self.info_msgs.append(str(msg)) + + def warning(self, msg, *_a, **_kw): + self.warning_msgs.append(str(msg)) + + +def _write_map(path, value, dtype=np.int16, n_covered=2): + """Build an integer map carrying ``value`` at the first ``n_covered``. + + Everything else reads the ``-1`` sentinel, i.e. off coverage. + """ + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, dtype, sentinel=-1 + ) + if n_covered: + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.full(n_covered, value, dtype=dtype), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_bool_map(path, n_covered=2): + """Build a boolean map, ``True`` at the first ``n_covered``.""" + smap = healsparse.HealSparseMap.make_empty( + NSIDE_COVERAGE, NSIDE_SPARSE, np.bool_ + ) + smap.update_values_pos( + RA[:n_covered], + DEC[:n_covered], + np.ones(n_covered, dtype=np.bool_), + lonlat=True, + ) + smap.write(str(path)) + return str(path) + + +def _write_sexcat(path, n=None): + """Write a synthetic LDAC SExtractor catalogue of known positions. + + Written with astropy rather than ``FITSCatalogue.save_as_fits``, because + creating an LDAC catalogue through that API requires an existing LDAC file + to copy the ``LDAC_IMHEAD`` HDU from. The three-HDU layout below is what + SExtractor writes and what ``SEx_catalogue=True`` (``hdu_no=2``) indexes. + """ + n = len(RA) if n is None else n + imhead = fits.BinTableHDU.from_columns( + [ + fits.Column( + name="Field Header Card", format="1A", array=np.array(["x"]) + ) + ], + name="LDAC_IMHEAD", + ) + objects = fits.BinTableHDU.from_columns( + [ + fits.Column(name="NUMBER", format="J", array=np.arange(n)), + fits.Column(name="XWIN_WORLD", format="D", array=RA[:n]), + fits.Column(name="YWIN_WORLD", format="D", array=DEC[:n]), + fits.Column( + name="IMAFLAGS_ISO", format="J", array=np.zeros(n, dtype="i4") + ), + ], + name="LDAC_OBJECTS", + ) + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto( + str(path), overwrite=True + ) + return str(path) + + +def _read(path): + cat = file_io.FITSCatalogue(str(path), SEx_catalogue=True) + cat.open() + data = cat.get_data() + flag = np.copy(data["MASK_EXT"]) + names = set(data.dtype.names) + cat.close() + return flag, names + + +def test_parse_map_paths(): + """Comma-separated paths parse, whitespace-tolerant, empties dropped.""" + assert mask_query_util.parse_map_paths( + " /a/star.hsp, /b/maximask.hsp ,, " + ) == ["/a/star.hsp", "/b/maximask.hsp"] + + +def test_flag_positions_integer_map(tmp_path): + """An integer map contributes its value; off-coverage stays clean.""" + path = _write_map(tmp_path / "m.hsp", 4, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [4, 4, 4, 0] + ) + + +def test_flag_positions_bits_restrict(tmp_path): + """MASK_BITS selects which bits of an integer map flag.""" + path = _write_map(tmp_path / "m.hsp", 1028, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=4), [4, 4, 4, 0] + ) + # A bit the map does not carry leaves everything clean. + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC, bits=2), [0, 0, 0, 0] + ) + + +def test_flag_positions_ors_maps(tmp_path): + """Several maps combine with a bitwise OR.""" + a = _write_map(tmp_path / "a.hsp", 4, n_covered=1) + b = _write_map(tmp_path / "b.hsp", 1024, n_covered=3) + npt.assert_array_equal( + mask_query_util.flag_positions([a, b], RA, DEC), [1028, 1024, 1024, 0] + ) + + +def test_flag_positions_boolean_map(tmp_path): + """A boolean map flags with 1; its False sentinel stays clean.""" + path = _write_bool_map(tmp_path / "bool.hsp", n_covered=2) + npt.assert_array_equal( + mask_query_util.flag_positions([path], RA, DEC), [1, 1, 0, 0] + ) + + +def test_mask_query_writes_flag_ext(tmp_path): + """The module writes MASK_EXT into a new catalogue and counts the hits.""" + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-0.fits") + out_path = tmp_path / "sexcat_ext-000-0.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=_NullLogger() + ).process() + + assert n_flagged == 2 + flag, names = _read(out_path) + npt.assert_array_equal(flag, [4, 4, 0, 0]) + assert np.issubdtype(flag.dtype, np.integer) + # The columns SExtractor wrote survive alongside the new one. + assert {"NUMBER", "XWIN_WORLD", "YWIN_WORLD", "IMAFLAGS_ISO"} <= names + # The LDAC structure survives: setools and psfex read HDU 2 by index. + with fits.open(str(out_path)) as hdus: + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + + # The input is not mutated: this module publishes a new file. + with fits.open(in_path) as hdus: + assert "MASK_EXT" not in hdus[2].data.dtype.names + + +def test_mask_query_bits_and_all_clean(tmp_path): + """MASK_BITS reaches the module, and a miss leaves every object clean.""" + map_path = _write_map(tmp_path / "m.hsp", 1024, n_covered=3) + in_path = _write_sexcat(tmp_path / "sexcat-000-1.fits") + out_path = tmp_path / "sexcat_ext-000-1.fits" + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], bits=4, w_log=_NullLogger() + ).process() + + assert n_flagged == 0 + flag, _ = _read(out_path) + npt.assert_array_equal(flag, [0, 0, 0, 0]) + + +def test_partial_read_matches_full(tmp_path): + """The partial read returns exactly what a full read of the map returns. + + This is the guarantee that makes the optimisation safe: query_map loads + only the coverage pixels the positions touch, and must be indistinguishable + from HealSparseMap.read(path) at every queried position. + """ + for dtype, writer in ((np.int16, _write_map), (np.bool_, None)): + path = ( + _write_map(tmp_path / f"full_{dtype.__name__}.hsp", 7, n_covered=3) + if writer is not None + else _write_bool_map(tmp_path / "full_bool.hsp", n_covered=3) + ) + full = healsparse.HealSparseMap.read(path) + expected = np.asarray(full.get_values_pos(RA, DEC, lonlat=True)) + npt.assert_array_equal( + mask_query_util.query_map(path, RA, DEC), expected + ) + + +def test_coverage_reported_for_both_map_kinds(tmp_path): + """in_coverage is the coverage mask, not valid_mask. + + For a boolean map healsparse stores only the True pixels, so valid_mask + would equal the value and could not distinguish an unmasked object from one + the map does not reach. The distant object must read as off-coverage while + the near, unflagged ones read as covered. + """ + for path in ( + _write_map(tmp_path / "int.hsp", 4, n_covered=2), + _write_bool_map(tmp_path / "bool.hsp", n_covered=2), + ): + _, in_coverage = mask_query_util.query_map_coverage(path, RA, DEC) + # The 4th position is 190 deg away; the first two are on the map. + assert in_coverage[0] and in_coverage[1] + assert not in_coverage[3] + + +def test_all_off_coverage_warns(tmp_path): + """A map that reaches nothing warns, instead of logging a silent zero.""" + path = _write_map(tmp_path / "elsewhere.hsp", 4, n_covered=2) + far_ra = np.array([200.0, 201.0]) + far_dec = np.array([-40.0, -41.0]) + log = _NullLogger() + + flag = mask_query_util.flag_positions([path], far_ra, far_dec, w_log=log) + + npt.assert_array_equal(flag, [0, 0]) + assert any("NO object is inside" in m for m in log.warning_msgs) + assert any("2 outside coverage" in m for m in log.info_msgs) + # A map that DOES reach the objects must not warn. + log2 = _NullLogger() + mask_query_util.flag_positions([path], RA, DEC, w_log=log2) + assert not log2.warning_msgs + + +def test_flag_positions_empty_input(tmp_path): + """Zero positions is not an error and reads no map.""" + flag = mask_query_util.flag_positions( + [str(tmp_path / "does-not-exist.hsp")], np.zeros(0), np.zeros(0) + ) + assert flag.shape == (0,) + + +def test_mask_query_empty_ccd(tmp_path): + """A CCD with no detections still publishes a catalogue, not an error. + + setools tolerates sparse-CCD attrition and psfex_interp's completeness + floor is 0/warn, so an empty sexcat must flow through rather than raise. + """ + map_path = _write_map(tmp_path / "star.hsp", 4, n_covered=2) + in_path = _write_sexcat(tmp_path / "sexcat-000-2.fits", n=0) + out_path = tmp_path / "sexcat_ext-000-2.fits" + log = _NullLogger() + + n_flagged = MaskQuery( + in_path, str(out_path), [map_path], w_log=log + ).process() + + assert n_flagged == 0 + assert out_path.exists() + flag, names = _read(out_path) + assert flag.shape == (0,) + assert "MASK_EXT" in names + assert any("No detections" in m for m in log.info_msgs) + + +def test_empty_coverage_raises_clearly(tmp_path): + """A map with no coverage at all fails with a message naming the file. + + The probe read in query_map_coverage indexes the first covered pixel; with + nothing covered that would be an IndexError from deep inside the utility. + """ + path = _write_map(tmp_path / "empty.hsp", 4, n_covered=0) + with pytest.raises(ValueError): + mask_query_util.query_map_coverage(path, RA, DEC) + + +def test_mask_query_no_maps_is_noop(tmp_path): + """No MASK_PATHS means a strict pass-through, not a zero column. + + The shipped configs comment MASK_PATHS out, so this is the DEFAULT path. + mask_query stays in the MODULE chain either way — setools reads its output + — so the no-op has to publish a file, and that file must be identical to + its input with no MASK_EXT column at all (the gating make_cat gives + MASK_EXT_PATHS). + """ + in_path = _write_sexcat(tmp_path / "sexcat-000-3.fits") + out_path = tmp_path / "sexcat_ext-000-3.fits" + log = _NullLogger() + + n_flagged = MaskQuery(in_path, str(out_path), [], w_log=log).process() + + assert n_flagged == 0 + assert out_path.exists() + assert out_path.read_bytes() == pathlib.Path(in_path).read_bytes() + with fits.open(str(out_path)) as hdus: + assert "MASK_EXT" not in hdus[2].data.dtype.names + assert [hdu.name for hdu in hdus] == [ + "PRIMARY", + "LDAC_IMHEAD", + "LDAC_OBJECTS", + ] + assert any("No MASK_PATHS configured" in m for m in log.info_msgs) + + +def test_flag_positions_no_maps(tmp_path): + """The utility with no paths returns an all-zero flag and reads nothing.""" + flag = mask_query_util.flag_positions([], RA, DEC) + npt.assert_array_equal(flag, [0, 0, 0, 0]) diff --git a/uv.lock b/uv.lock index a4d20ff52..e1d016ab6 100644 --- a/uv.lock +++ b/uv.lock @@ -141,24 +141,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/ef/a5cf36a402c4511a776405f12d0b35196f55f4312ce407012a2fbcf1e4e0/astropy-8.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b482bc6c57c966e6c6234410a1d9afbcf92bcac858cf287812f8c99ddc3fafc", size = 10407732, upload-time = "2026-07-05T07:24:40.774Z" }, ] -[[package]] -name = "astropy-healpix" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/c1/aeb3fe3be2ee863708d625014267c71abfe20ddaa293b3d4ddb72ee1d6e9/astropy_healpix-2.0.1.tar.gz", hash = "sha256:0e3f1c94064c45da779900cb90c938df7aef99a924abb23eeb893b16540e77e6", size = 112256, upload-time = "2026-07-20T21:07:30.004Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/3d/0cde0db89ac8dd4e5347322470530298915739f7e9b356b01c1939de8c4f/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:179119d5a69e7b9245919cbe04c3e6bf0a485516b36c29f3402951aad5452251", size = 191178, upload-time = "2026-07-20T21:07:16.512Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c4/71cf2bd4374cc17e015413462be8051fe08bc49e077b7d48072fff4e465d/astropy_healpix-2.0.1-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c092c54124c48f8d98e04fb22f4b2aa4c8675e65d81c351523f41377f9a6df22", size = 193429, upload-time = "2026-07-20T21:07:18.015Z" }, - { url = "https://files.pythonhosted.org/packages/fa/5c/3a50f68225b836b395da4fb8dfd3d702ded1916042490b16a613caa0e4a6/astropy_healpix-2.0.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce875a29c598c1a99f8f68351daeb4173463044dce4f1c7ebfe8c233ec5e9a49", size = 188694, upload-time = "2026-07-20T21:07:19.244Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a7/c369703ca3fc6f5bee31b3d1f6d0f0b38c15ec84e7fbb6f552c0c7cedfb2/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78f76785852bcc748f5efab8bb4f2ab1fe959d7a998b48e7ed1e59a46cbf0e51", size = 198717, upload-time = "2026-07-20T21:07:24.657Z" }, - { url = "https://files.pythonhosted.org/packages/32/d5/0c4183611b8f36877112882e25cc8a91655a2d11e120ea1f5c153cb7d3a0/astropy_healpix-2.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02cefde735da1fe74654e786e02286261b04cc43f5c908a86a8457b2989ac2aa", size = 201205, upload-time = "2026-07-20T21:07:26.128Z" }, - { url = "https://files.pythonhosted.org/packages/e2/54/42bcd02b7c604d3a1132dfa93195bdc2677bb7844172cd098e149cbcb4e9/astropy_healpix-2.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65bee7b70f35ddb81d6b6c849c5bf46768afbce6869395e4f9e0a5c27cb0ce17", size = 196134, upload-time = "2026-07-20T21:07:27.55Z" }, -] - [[package]] name = "astropy-iers-data" version = "0.2026.8.10.0.32.39" @@ -168,24 +150,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/27/0775134a2a939ddf7a073e77e66110b671e99cefbc36fb57b57851fd109a/astropy_iers_data-0.2026.8.10.0.32.39-py3-none-any.whl", hash = "sha256:04239afd6e9615165b84da1b84d73f03ca473f0ddf53b516c260e27349a95d03", size = 1998788, upload-time = "2026-08-10T00:33:34.729Z" }, ] -[[package]] -name = "astroquery" -version = "0.4.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "beautifulsoup4" }, - { name = "html5lib" }, - { name = "keyring" }, - { name = "numpy" }, - { name = "pyvo" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/48/273dbde090e071f9d264d084bc49193d126498d2906172b78febd9d62e28/astroquery-0.4.11.tar.gz", hash = "sha256:5537529bddc7fa07e773d5cd9baca593e3f5d93474edd1914f68e89506042b33", size = 12561055, upload-time = "2025-09-20T04:26:36.744Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/ca/944b328f2c60b83896a9223312e21e46cdc1fef57565e853da4544ff6a8e/astroquery-0.4.11-py3-none-any.whl", hash = "sha256:e34f114b285dd07a10ddb2065ebce829b01b0e740fd89dbc81a3077808e24b2d", size = 11139417, upload-time = "2025-09-20T04:26:31.881Z" }, -] - [[package]] name = "asttokens" version = "3.0.2" @@ -494,15 +458,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -730,45 +685,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] -[[package]] -name = "dask" -version = "2026.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "cloudpickle" }, - { name = "fsspec" }, - { name = "packaging" }, - { name = "partd" }, - { name = "pyyaml" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/39/cbd21c9133d02b4e60899ed466ad5e876553ea68ebffee6209e7dcafc8d4/dask-2026.7.1.tar.gz", hash = "sha256:5727484427665f051e86bf87d021a64d6411141cdc8a20bfe3c1ad2968cc06b7", size = 11548794, upload-time = "2026-07-14T01:06:22.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/5f/7c22733da92b3a6cc4dddcaa8731089d213c2790bbc997e51c429a4e8f8b/dask-2026.7.1-py3-none-any.whl", hash = "sha256:985ffd6c5e9d7979ede515e84ae8d39b647d6aa64f77600f15714ff65f578fe6", size = 1496882, upload-time = "2026-07-14T01:06:20.341Z" }, -] - -[package.optional-dependencies] -array = [ - { name = "numpy" }, -] - -[[package]] -name = "dask-image" -version = "2026.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dask", extra = ["array"] }, - { name = "numpy" }, - { name = "pims" }, - { name = "scipy" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/5b/15d6d6ff8697b188787609be059fe4f07f99fc00f43f68e9e1540fa8733e/dask_image-2026.5.0-py3-none-any.whl", hash = "sha256:acf86cd7f0f1e97804198d30b7cc931efd29f9dec86c65ec004b50405c3f5227", size = 43814, upload-time = "2026-05-27T14:05:56.237Z" }, -] - [[package]] name = "datetime" version = "6.0" @@ -821,18 +737,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "donfig" -version = "0.8.1.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, -] - [[package]] name = "dpath" version = "2.2.0" @@ -913,15 +817,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, ] -[[package]] -name = "fsspec" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, -] - [[package]] name = "future" version = "1.0.0" @@ -978,20 +873,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/ed/ae57eb7d344f43f87b74b3a281ead6ec7d6394eef72a7b1dcb28dd089550/gitpython-3.1.59-py3-none-any.whl", hash = "sha256:67a82f537384578643624c8b2c531938a9b82be431663e575dcf638526631d4c", size = 220996, upload-time = "2026-08-10T12:03:18.804Z" }, ] -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, -] - [[package]] name = "greenlet" version = "3.5.5" @@ -1151,19 +1032,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, ] -[[package]] -name = "html5lib" -version = "1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173, upload-time = "2020-06-22T23:32:36.781Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" @@ -1840,15 +1708,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/f9/7b7b50f80b4585bcd78675ff3110c256877b11df32a8cde284f851762f57/llvmlite-0.49.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32adb84fdaae28aeb86fdb6253084ee707ee157289a2e98fe3caf48a62bee82", size = 58344482, upload-time = "2026-08-11T16:25:51.527Z" }, ] -[[package]] -name = "locket" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, -] - [[package]] name = "lsstdesc-coord" version = "1.3.1" @@ -2293,24 +2152,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/f9/3a7b6dbf81e01a48958b45ad2239edbc64707522ab17f11f9f18c44bf6d1/numba-0.67.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83ab968b0e0fa744eba03351282dd8000796e6ec8e4518f47bd3ed86c0a20c7b", size = 3614644, upload-time = "2026-08-11T23:03:55.794Z" }, ] -[[package]] -name = "numcodecs" -version = "0.16.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, - { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, - { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, - { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, - { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, -] - [[package]] name = "numpy" version = "2.5.2" @@ -2411,19 +2252,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] -[[package]] -name = "partd" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "locket" }, - { name = "toolz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, -] - [[package]] name = "pexpect" version = "4.9.0" @@ -2477,19 +2305,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, ] -[[package]] -name = "pims" -version = "0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imageio" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "slicerator" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } - [[package]] name = "platformdirs" version = "4.11.3" @@ -2584,15 +2399,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "pyavm" -version = "0.9.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/ac/a925d36dd37fc37f89afccc1f62ffa03b7344c8e4ff7850be7879b4497e6/pyavm-0.9.9.tar.gz", hash = "sha256:bc0f605d957c1fd6d7765523fcba8b9a72377ac6c51461c2a838fe44600bcd9a", size = 220572, upload-time = "2026-03-12T09:54:50.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/ab/ba8d2b40aee05cd986807d58ee324369eb16248b717a08b51eee977f8d33/pyavm-0.9.9-py3-none-any.whl", hash = "sha256:8bba0ee9645a8a9f215af9ceea67b494a6ee1fe380cebdc00ba99d781539ad76", size = 379786, upload-time = "2026-03-12T09:54:49.455Z" }, -] - [[package]] name = "pybind11" version = "3.1.0" @@ -2951,19 +2757,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, ] -[[package]] -name = "pyvo" -version = "1.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/35/12c0f4fa0879316837ac56f275942f006f0735e5aedba529b4449dddc36f/pyvo-1.9.1.tar.gz", hash = "sha256:2f26c99af7c32f3c34b919e2d14eaf1a95914176d693fb7769773f3ab0b7999d", size = 2167800, upload-time = "2026-06-11T14:44:01.349Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/09/04ff6e8beaa6cd60a960f925d837c2afa3fffba7860042ab1b52a8358297/pyvo-1.9.1-py3-none-any.whl", hash = "sha256:098648d00943440f56c1d00ac76330433578a0725ea6187afaa6bae08545bb53", size = 1152763, upload-time = "2026-06-11T14:43:59.016Z" }, -] - [[package]] name = "pywavelets" version = "1.9.0" @@ -3092,28 +2885,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] -[[package]] -name = "reproject" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astropy" }, - { name = "astropy-healpix" }, - { name = "dask", extra = ["array"] }, - { name = "dask-image" }, - { name = "fsspec" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pyavm" }, - { name = "scipy" }, - { name = "zarr" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/5b/8d9b51c754ab014194d374cb4873d0729b397f67997e67721538b643e682/reproject-0.21.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2132fc5d2fa3fbbd57337099f9d47582136e653ac82402783c7ceffa26804816", size = 1776590, upload-time = "2026-06-25T15:11:30.358Z" }, - { url = "https://files.pythonhosted.org/packages/da/2d/f9d76e8e308813978227e1e43a1f25f64cd0f4b030cecaeff8baeec9eeac/reproject-0.21.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8525baaca84e949a69532c02ff491c23993e3bc22a59f694af576e473f2a9d2f", size = 1791889, upload-time = "2026-06-25T15:11:31.741Z" }, -] - [[package]] name = "requests" version = "2.34.2" @@ -3442,11 +3213,12 @@ version = "1.1.0" source = { editable = "." } dependencies = [ { name = "astropy" }, - { name = "astroquery" }, { name = "canfar" }, { name = "cs-util" }, { name = "galsim" }, { name = "h5py" }, + { name = "healsparse" }, + { name = "hpgeom" }, { name = "joblib" }, { name = "matplotlib" }, { name = "mccd" }, @@ -3460,7 +3232,6 @@ dependencies = [ { name = "pyqtgraph" }, { name = "python-dateutil" }, { name = "python-pysap" }, - { name = "reproject" }, { name = "sf-tools" }, { name = "skaha" }, { name = "sqlitedict" }, @@ -3518,13 +3289,14 @@ test = [ [package.metadata] requires-dist = [ { name = "astropy", specifier = ">=7.0" }, - { name = "astroquery" }, { name = "build", marker = "extra == 'release'" }, { name = "canfar" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util?branch=develop" }, { name = "fitsio", marker = "extra == 'fitsio'" }, { name = "galsim", specifier = ">=2.8" }, { name = "h5py" }, + { name = "healsparse" }, + { name = "hpgeom" }, { name = "hypothesis", marker = "extra == 'test'", specifier = ">=6.155.2" }, { name = "ipython", marker = "extra == 'jupyter'", specifier = ">=9.14.1" }, { name = "joblib", specifier = ">=1.4" }, @@ -3545,7 +3317,6 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0" }, { name = "python-dateutil" }, { name = "python-pysap", specifier = ">=0.3" }, - { name = "reproject", specifier = ">=0.19" }, { name = "ruff", marker = "extra == 'lint'" }, { name = "sf-tools", specifier = ">=2.0.4" }, { name = "shapepipe", extras = ["doc", "jupyter", "lint", "release", "test", "fitsio"], marker = "extra == 'dev'" }, @@ -3598,15 +3369,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/d3/a3c1d569ae714ed9985adccefdfe2d6dd9f8ad7a297bd8fa4c7ed30c587b/skaha-1.7.0-py3-none-any.whl", hash = "sha256:68b0d3c925b98bf145c5f695237474fe1cea07cf9b69b4a9bae9910375dfa01a", size = 40795, upload-time = "2025-05-28T21:17:02.296Z" }, ] -[[package]] -name = "slicerator" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/52/f38586b82b2935f8b59a09b0a79c545a22ed062e728c9418bafeb51f61e0/slicerator-1.1.0.tar.gz", hash = "sha256:44010a7f5cd87680c07213b5cabe81d1fb71252962943e5373ee7d14605d6046", size = 38283, upload-time = "2022-04-07T18:54:08.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, -] - [[package]] name = "smart-open" version = "7.7.1" @@ -4069,15 +3831,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, ] -[[package]] -name = "toolz" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, -] - [[package]] name = "tornado" version = "6.5.8" @@ -4293,23 +4046,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, ] -[[package]] -name = "zarr" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "donfig" }, - { name = "google-crc32c" }, - { name = "numcodecs" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/15/436cb1d3bbe86173bd44ce7a34ecb210d0c0416946e337858149a905ef5a/zarr-3.3.0.tar.gz", hash = "sha256:cd0c8cf738b4bb4807815bc1255acad5bdf1a7b7264b606c5a1bc0d0392a306b", size = 943626, upload-time = "2026-07-30T16:35:10.491Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/c6/6b726ddf4c3ac5a123f285c3650fa8268902c28ea541a0282867f1336e65/zarr-3.3.0-py3-none-any.whl", hash = "sha256:323bf5366d4f909052ef6e2e03e7481a7434c3ee75d3a981eb3a71fc1ae22cef", size = 363685, upload-time = "2026-07-30T16:35:08.794Z" }, -] - [[package]] name = "zipp" version = "4.1.0" diff --git a/workflow/README.md b/workflow/README.md index 08bfd2291..5ace34019 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,8 +22,7 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the -# star-catalogue cache root). +# Edit workflow/config.yaml: tile_list, run_dir, container. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -154,8 +153,8 @@ workflow/ bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures - exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) - tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + exposure.smk per-exposure: get_images, split, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) build_index.py prepare-phase run_index.sqlite builder (plain script) @@ -196,16 +195,24 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee committed under `workflow/config/cfis/` and version with the rules that set the env vars they interpolate — there is no `config_src` knob, and no per-unit config symlink; `$SP_CONFIG` points straight at the committed directory. -- **Mask star catalogues are built in the DAG.** `exp_star_cat` runs one Vizier - cone query per exposure into the run-independent cache at `star_cats:`, then - fans it out into a real per-unit `star_cat_exp/` directory of 40 per-CCD - symlinks, which `exp_mask` consumes. The directory must be per-unit and real: - the file handler intersects the image numbers it finds across a config's - `INPUT_DIR`s, so a symlink to the whole cache contributes every other - exposure's numbers and the intersection comes out empty. It is a `localrule`, - so the queries run serially in the head process — CDS is never hammered, and - the scheduler never sees a six-second job. The cache makes reruns and later - campaigns free. +- **There is no masking stage, on either side.** ShapePipe generates no masks + (PR #847). The one mask that reaches pixels is the instrument flag image + delivered with each exposure, which `exp_split` splits per CCD beside image + and weight and SExtractor reads as `IMAFLAGS_ISO`. Everything else — star + halos, manual masks, per-band coverage, MaxiMask — is supplied as sky-fixed + healsparse maps and QUERIED once per object: the `mask_query` module writes a + `MASK_EXT` column onto each CCD's detection catalogue (inside `exp_psf`), and + `make_cat` writes one `MASK_` column per band onto the final catalogue + (inside `tile_make_cat`). Neither is cut on in the pipeline: instrument flags + mark corrupted measurements and are the only masks that reject anything here, + while the healsparse masks are location flags and every decision about them is + downstream. On exposures the query ships off — `MASK_PATHS` is commented out, + making `mask_query` a no-op pass-through — so turning it on is a config edit, + not a chain edit. Map paths are config, not + code, so regenerated products cost a config edit. Nothing is fetched from a + catalogue server, staged, or rasterized, which is why the old + `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are + gone. - **The index is parse-time data, never a rule input.** Appending tiles changes which jobs exist without invalidating completed work. - **Exposure products are not `temp()`.** Exposures overlap tiles, so diff --git a/workflow/Snakefile b/workflow/Snakefile index b925adc06..8a8a2a9bf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -82,10 +82,6 @@ RUN_DIR = Path(config["run_dir"]) # Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no # second path: one root, exactly the pre-D5 layout. PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) -# Run-independent root for the mask star catalogues: the HEALPix chunk store the -# star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it -# (config.yaml explains the placement). -STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the @@ -309,13 +305,6 @@ SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") -# Same argument for star_cats.py, which both star-cat rules call: their params -# otherwise fingerprint nothing but paths, so an edit to the chunking or the cut -# would never rerun them. ONE hash for both rules because it is one script — and -# that is also why fetch and cut live in one module (they must agree on which -# pixel holds which star). The hash does NOT key the store path — see -# config.yaml's star_cats block on clearing the store after a semantic change. -STAR_CAT_HASH = script_hash("star_cats.py") # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -509,8 +498,7 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale pre-fix file with the bare id. There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at - the committed config dir, and ``star_cat_exp`` is a real per-unit directory - built by the ``exp_star_cat`` rule, not a symlink into a shared pool. + the committed config dir. Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -589,18 +577,16 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# The aggregation targets, clean_exposure, clean_tile, star_catalogue and -# exp_star_cat run in the head process. The two clean rules are seconds of rmtree -# and hang off `all`; exp_star_cat is seconds of local FITS work; all three would -# otherwise be ~20k (clean_exposure, exp_star_cat) or ~23k (clean_tile) sbatch -# submissions at DR6 scale for work shorter than the scheduling latency. -# star_catalogue is one job either way, and local keeps its CDS concurrency the -# explicit number its thread pool sets (see exposure.smk). +# The aggregation targets, clean_exposure and clean_tile run in the head +# process. Both clean rules are seconds of rmtree and hang off `all`; submitted +# they would be ~20k (clean_exposure) or ~23k (clean_tile) sbatch submissions at +# DR6 scale for work shorter than the scheduling latency. # -# star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out -# of any future `group:` label: a local job cannot be fused into a submitted group. -# The two clean rules are DAG leaves and have no such constraint. -localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat +# Both are DAG LEAVES, so neither constrains a `group:` label. (A mid-chain +# localrule would: a local job cannot be fused into a submitted group. The old +# star-catalogue rules were exactly that, and they are gone with the internal +# mask generation.) +localrules: all, prepare_all_tiles, clean_exposure, clean_tile rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index 8e4de8506..e3f0355aa 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -53,30 +53,6 @@ products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# The mask star-catalogue root — run-independent, shared by every campaign, and -# holding two things: -# /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 -# query per HEALPix chunk (~3.4 deg^2, ~25k rows), written by -# `star_catalogue` over the tile list's footprint and never fetched twice; -# /exp/star_cat-.fits the per-exposure cuts -# `exp_star_cat` makes from those chunks, with no network at all. -# Network therefore scales with SKY AREA, not exposure count: exposures overlap -# ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. -# -# On the PERSISTENT root: the sky store is a durable science product bought with -# ~1.5k catalogue-server queries at DR6 scale, and re-buying it after a scratch -# purge is the one cost in this workflow that cannot be paid with local compute. -# (It sat on scratch through smk-g3 only because def-mjhudson /project was then -# hard-full at 27/27 TiB.) -# -# THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to -# workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun -# both rules — the script's hash is a param on each — but a chunk already on disk -# is skipped and only re-cut. Clear the store by hand when the change must reach -# the data. Changing NSIDE or the catalogue ID is the exception: those name the -# directory, so a change there fetches into a new one beside the old. -star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache - # The run index, and — sharing its directory — missing.json and run_report.json. # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction @@ -142,7 +118,7 @@ clean_tiles: true # # READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its # exposures' stores. If you later retry that tile, those exposure chains are -# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# gone and will be REBUILT from scratch — get_images, split, psf, per # exposure. That is correct, and expensive. Ignore a tile when you have decided # it is dead, not while you are still debugging it. clean_ignore_tiles: [] diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini deleted file mode 100644 index d5b521080..000000000 --- a/workflow/config/cfis/config_exp_Ma.ini +++ /dev/null @@ -1,86 +0,0 @@ -# ShapePipe configuration file for masking of exposures - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_exp_Ma - -# Add date and time to RUN_NAME, optional, default: False -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = mask_runner - -# Parallel processing mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# Input directory, containing input files, single string or list of names -INPUT_DIR = . - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 4 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -### Mask exposures -[MASK_RUNNER] - -# Parent module -INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp - -# Update numbering convention, accounting for HDU number of -# single-exposure single-HDU files -NUMBERING_SCHEME = -0000000-0 - -# Input file patterns: image, weight, external flag, external star catalogue -FILE_PATTERN = image, weight, flag, star_cat - -FILE_EXT = .fits, .fits, .fits, .fits - -# Path of mask config file -MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask - -# External mask file flag, use if True, otherwise ignore -USE_EXT_FLAG = True - -# External star catalogue flag, use external cat if True, -# obtain from online catalogue if False -# True: the cat comes from $SP_RUN/star_cat_exp, the per-unit farm the -# exp_star_cat rule builds (40 per-CCD links to this exposure's one cat). -USE_EXT_STAR = True - -# File name suffix for the output flag files (optional) -PREFIX = pipeline - -# Path to check for existing output mask files -CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini index 0af871d8e..92b230739 100644 --- a/workflow/config/cfis/config_exp_psfex.ini +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -1,5 +1,8 @@ # ShapePipe configuration file for single-exposures. PSFex PSF model. -# Process exposures after masking, from star detection to PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -20,8 +23,7 @@ RUN_DATETIME = False [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner - +MODULE = sextractor_runner, mask_query_runner, setools_runner, psfex_runner, psfex_interp_runner # Run mode, SMP or MPI MODE = SMP @@ -57,11 +59,11 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Input from two modules -INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag # Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on # the decorator's 4-entry FILE_EXT default (length check fails at startup) @@ -126,16 +128,50 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets MASK_EXT. Comma-separated. +# +# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue +# is passed through with no MASK_EXT column, the same gating make_cat gives +# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning +# the query on is uncommenting one line and never editing the chain. +# +# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive +# (instrument flags only) and the column is carried for transparency and +# measurement — see star_selection.setools' header for the one-line change +# that would impose it. +; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/mask_query_runner/output + +FILE_PATTERN = sexcat_ext + +NUMBERING_SCHEME = -0000000-0 + # SETools config file SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask deleted file mode 120000 index 9cd6a04b1..000000000 --- a/workflow/config/cfis/config_onthefly.mask +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/config_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 0ce9ea226..5487f547c 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -101,7 +101,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask deleted file mode 120000 index 4743a87f0..000000000 --- a/workflow/config/cfis/config_tile_onthefly.mask +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/config_tile_onthefly.mask \ No newline at end of file diff --git a/workflow/config/cfis/mask_default b/workflow/config/cfis/mask_default deleted file mode 120000 index 0970152ab..000000000 --- a/workflow/config/cfis/mask_default +++ /dev/null @@ -1 +0,0 @@ -../../../example/cfis/mask_default \ No newline at end of file diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 9fe3e7b29..af60dadbe 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,17 +1,22 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -----> exp_mask -> exp_psf - -> exp_star_cat --/ - star_catalogue ---------------/ - -``star_catalogue`` is campaign-level, not per-exposure: one fetch of the whole -footprint's stars, which every exposure's ``exp_star_cat`` then cuts locally. + exp_get_images -> exp_split -> exp_psf Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a run log. There is no `prepare_exposures` aggregation target: these chains hang off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). +NO MASK RULE, and that is the design (PR #847). ShapePipe generates no masks. +The only mask that reaches pixels is the instrument flag image delivered with +the exposure, which ``exp_split`` splits per CCD alongside image and weight and +SExtractor reads directly. Sky-fixed masks are healsparse maps, queried once per +object: ``mask_query`` (inside exp_psf's config chain) writes ``FLAG_EXT`` onto +each CCD's SExtractor catalogue for setools' star cut, and ``make_cat`` writes +the per-band ``MASK_`` columns on the tile side. Neither needs a rule, a +star catalogue, or a network fetch — hence no ``star_catalogue`` / ``exp_star_cat`` +here, and no ``exp_mask``. + NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, not over one invocation — reclamation here is clean_exposure's job (S5), driven @@ -19,26 +24,21 @@ by the accumulating index. A temp() here would delete an exposure the moment this invocation's readers finished and cascade destructive reruns across spatial neighbours the next time a tile is appended. -GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — -one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under -the 15-minute floor Alliance policy asks us to bundle away. The composition -rules are in prepare.smk's docstring; this chain is linear too, so the group -asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. - -The two rules NOT in it are structural, not taste: - * exp_psf is heavy (16 GB, 4 h) and never fuses with a short rule; - * exp_get_images cannot join, because ``exp_star_cat`` — a LOCALRULE, and so - ungroupable — sits between it and exp_mask. Pulling get_images in would make - the group both a dependency and a dependent of exp_star_cat, i.e. a cycle. - Starting the group at exp_split leaves star_cat's inputs entirely upstream - of it, so the group has one clean external edge. -Different exposures share no DAG edge, so this is one group job per exposure. +NO GROUPING. The ``exp_short`` group existed to fuse exp_split and exp_mask — +two rules whose medians were 1:28 and 1:54, both well under the 15-minute floor +Alliance policy asks us to bundle away — into one sbatch per exposure. With +exp_mask gone there is nothing to fuse: a group of one rule submits exactly the +job the ungrouped rule submits, and the label would only obscure that. The +composition rules, should a second short rule ever appear here, are in +prepare.smk's docstring. exp_get_images stays separate for the same reason it +always did (a download, retried on its own), and exp_psf is heavy (16 GB, 4 h) +and never fuses with a short rule. NUMBER_LIST ($SP_UNIT_NUM, see unit_num in the Snakefile) is set only for exp_split, whose numbering scheme IS the exposure id; never for get_images / -exp_mask / exp_psf, whose per-CCD or download numbering would turn tolerated -per-CCD attrition into a whole-exposure hard failure. It is a property of the -committed configs (config_exp_Sp.ini alone carries the entry). +exp_psf, whose per-CCD or download numbering would turn tolerated per-CCD +attrition into a whole-exposure hard failure. It is a property of the committed +configs (config_exp_Sp.ini alone carries the entry). """ rule exp_get_images: @@ -58,198 +58,9 @@ rule exp_get_images: shell: sp_shell("exp_get_images", "config_exp_Gie.ini") -# --- mask star catalogues --------------------------------------------------- -# Two rules, and the split between them is the design: the NETWORK is a function -# of the campaign's sky area, the per-exposure catalogue is a local cut. -# -# `star_catalogue` fetches the footprint's GSC 2.3 stars once, one Vizier query -# per HEALPix chunk, into a run-independent chunk store under config `star_cats`. -# `exp_star_cat` then reads the chunks covering an exposure's focal plane and -# cuts them to it — no network at all. workflow/scripts/star_cats.py holds both -# halves, the geometry they must agree on, and the arithmetic that motivates the -# split; its docstring is the reference for chunking, padding and query counts. - -# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE -# pointing at a path that does not exist inside the image, so requests is pointed -# at the bundle explicitly (proven in the p3-batch1 bash precedent). -STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" - -# The rules run star_cats.py inside the container (healpy, astroquery, astropy) -# but call apptainer THEMSELVES rather than letting the SDM wrap them -# (`container: None` on both): the CA bundle above and the exposure rule's -# host-side farm loop both need the explicit exec. bin/sp has loaded the -# apptainer module. -# -# WHICH image and WHICH arguments are not this file's to decide, and hand-rolling -# them here was a real divergence: these were the only two rules that ignored a -# user's dev sandbox, because they read `config['container']` — the shared /project -# fallback — instead of the image the Snakefile resolved for everything else. -# `_image` is that resolution (sandbox -> cached SIF -> config), and the profile's -# own apptainer-args are the same string the SDM splices onto every other rule, -# PYTHONPATH pin for shapepipe.utilities.{vizier,cfis} included. -# container.profile_apptainer_args() exists precisely so this file can read them -# rather than restate them. Only the CA bundle is added on top, and only when the -# command touches the network. -def in_container(cmd, *, network=False): - args = list(_container.profile_apptainer_args()) - if not args: - # Silently falling back would run these two rules with no --cleanenv and - # no PYTHONPATH pin, i.e. against a different src/ than every other rule. - raise WorkflowError( - f"Could not read apptainer-args from {_container.PROFILE_FILE}; " - f"star_catalogue and exp_star_cat build their apptainer line from it.") - if network: - # The container's certifi bundle, one --env per variable (the profile's - # own PYTHONPATH entry uses the same one-assignment-per-flag form). - args += [a for k in ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", - "CURL_CA_BUNDLE") - for a in ("--env", f"{k}={STAR_CAT_CA}")] - return f"apptainer exec {' '.join(args)} '{_image}' {cmd}" - - -# The campaign's star catalogue: a first-class durable science product, keyed by -# sky rather than by run. Chunk-need is recomputed from the tile list on every -# run and only the missing chunks are fetched, so appending tiles costs exactly -# the chunks they add. -# -# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the -# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the -# per-exposure rule reached by accident through --local-cores, now an explicit -# number that does not scale with the head node's CPU count. -# -# `tile_list_hash` is what makes the incremental behaviour visible to the DAG. -# The tile list is parse-time config, not a rule input (and the profile drops the -# `input` rerun-trigger anyway), so appending tiles would otherwise leave this -# rule up to date against a footprint that has grown. Hashing the list into a -# param reruns it, and the rerun fetches only what is new. -STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" - - -rule star_catalogue: - output: - manifest = STAR_CAT_MANIFEST - # No `log:` — see write_manifest() in star_cats.py. - # `cmd` is a params value, so placeholders in it are not formatted (see - # unit_pre in the Snakefile). Hence the explicit manifest path. - params: - cmd = in_container( - f"python {SCRIPTS}/star_cats.py fetch" - f" --tile-list '{config['tile_list']}' --store '{STAR_CATS}'" - f" --manifest '{STAR_CAT_MANIFEST}'", network=True), - tile_list_hash = hashlib.md5( - Path(config["tile_list"]).read_bytes()).hexdigest()[:12], - script_hash = STAR_CAT_HASH - container: - None - threads: 4 - retries: 2 - resources: - mem_mb = 4000, - runtime = 720 - shell: - "set -euo pipefail\n{params.cmd}" - - -# The per-exposure catalogue and the 40 per-CCD symlinks the mask module's -# numbering scheme needs. Local: one header read for the focal-plane footprint, -# a load of the chunks covering it, a radial cut. -# -# A LOCALRULE, for the reason the Snakefile's localrules line gives. -# -# The per-unit farm is a REAL directory holding exactly this exposure's 40 -# numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR -# and the file handler INTERSECTS the numbers found across INPUT_DIRs, so a -# symlink to a shared whole-store pool contributes every other exposure's numbers -# and the intersection is empty ("numbers ... do not intersect", live). -# -# TWO declared outputs, and the second one is the point. -# -# The manifest keeps the "one rule, one manifest" currency of every other rule: -# written last, unique to this rule, a record of what the farm points at, and -# deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the -# chunk store at no network cost. -# -# But a manifest attests FOREVER, and the two things it attests to both live -# outside the unit's manifests/ dir: the cut catalogue on /scratch (60-day purge) -# and the farm itself. Either can vanish under a manifest that still says -# "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is -# declared too — one link stands for all 40, they are created by the same loop -# in the same instant, and declaring 40 buys nothing. Snakemake's existence test -# is os.path.exists, which FOLLOWS symlinks and is therefore False for a link -# whose target the purge removed. A purged cut or a deleted farm makes the rule -# out of date, it reruns, and it re-cuts or re-links as needed. -def star_cat_cmd(exp): - """The whole rule body, as bash — carried as a params value because it - contains literal ``{}`` (the manifest JSON); see unit_pre in the Snakefile.""" - cut_dir = f"{STAR_CATS}/exp" - cat = f"{cut_dir}/star_cat-{exp}.fits" - work = exp_dir(exp) - farm = f"{work}/star_cat_exp" - images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" - manifest = exp_manifest(exp, "exp_star_cat") - body = json.dumps({ - "stage": "exp_star_cat", "level": "exp", "unit": exp, - "status": "complete", "cat": cat, "link_dir": farm, "n_links": 40, - }, indent=2, sort_keys=True) - return "\n".join([ - "set -euo pipefail", - # LEGACY-SYMLINK HAZARD. Unit dirs built before this rule existed carry - # star_cat_exp as a SYMLINK into the old shared star-cat pool. `mkdir -p` - # is a no-op on an existing symlink-to-directory, so the 40-link loop - # below followed it and wrote this exposure's links INTO THE SHARED POOL - # (520 stray links found live). Replace the link — never `rm -rf` it, - # which would recurse into the pool, and never touch a real directory: - # a real farm is this rule's own output and `ln -sfn` refreshes it. - f"[ -L '{farm}' ] && rm -f '{farm}' || true", - f"mkdir -p '{cut_dir}' '{farm}' '{work}/manifests'", - in_container(f"python {SCRIPTS}/star_cats.py cut" - f" --images '{images}' --store '{STAR_CATS}'" - f" --out '{cat}'"), - f"test -s '{cat}'", - # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the - # one focal-plane catalogue (pattern from the p3-batch1 precedent). - f"for ccd in $(seq 0 39); do ln -sfn '{cat}' " - f"'{farm}/star_cat-{exp}-'\"$ccd\"'.fits'; done", - # Byte-stable, and written only after the links exist: an unconditional - # write would move the mtime, which is a rerun-trigger. - f"tmp='{manifest}.tmp'", - "cat > \"$tmp\" <<'SP_STAR_CAT_JSON'", - body, - "SP_STAR_CAT_JSON", - f"cmp -s \"$tmp\" '{manifest}' && rm -f \"$tmp\" || mv -f \"$tmp\" '{manifest}'", - ]) - - -rule exp_star_cat: - input: - rules.exp_get_images.output.manifest, - # The chunks this cut reads. star_cats.py fails loudly on a chunk that is - # missing anyway, but the edge is what makes the fetch happen first. - rules.star_catalogue.output.manifest - output: - manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", - # The sentinel: ccd-0 of the 40-link farm (see above). - link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" - # No `log:`, for the same reason as star_catalogue above. - params: - cmd = lambda wc: star_cat_cmd(wc.exp), - # star_cats.py is external to the shell string, so the `code` - # rerun-trigger does not see it — same reason SCRIPT_HASH exists. - script_hash = STAR_CAT_HASH - container: - None - threads: 1 - retries: 2 - resources: - mem_mb = 4000, - runtime = 10 - shell: - "{params.cmd}" - # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: - group: "exp_short" input: rules.exp_get_images.output.manifest output: @@ -266,33 +77,13 @@ rule exp_split: shell: sp_shell("exp_split", "config_exp_Sp.ini") -rule exp_mask: - group: "exp_short" - input: - # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs - # and this exposure's own star_cat_exp farm. - rules.exp_split.output.manifest, - rules.exp_star_cat.output.manifest - output: - manifest = f"{EXP_DIR}/manifests/exp_mask.json" - log: - f"{EXP_DIR}/logs/exp_mask.json" - params: - pre = lambda wc: unit_pre("exp_mask", wc.exp), - script_hash = SCRIPT_HASH - threads: 4 - resources: - mem_mb = lambda wc, attempt: 8000 * attempt, - runtime = 120 - shell: - sp_shell("exp_mask", "config_exp_Ma.ini") - -# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# SExtractor -> mask_query (FLAG_EXT) -> setools star selection -> PSFEx model +# -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's # :warn on psfex_interp_runner. rule exp_psf: input: - rules.exp_mask.output.manifest + rules.exp_split.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_psf.json" log: diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 07405407f..fdf5d726b 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -29,11 +29,12 @@ cached group resources and re-sets ``attempt`` on every member (jobs.py), and ``retries: 2`` still governs. A retry re-runs the whole group, which is safe because every rule ``rm -rf``s its own run dir at start. -Star catalogues for masking are NOT a prepare-phase concern and not pre-run -input: the compute DAG fetches the campaign footprint's stars once -(``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in -exposure.smk, into a run-independent store. The tile side has no star-cat node -because it has no mask rule yet — see tile.smk. +There is no masking node in this phase, or in any other: ShapePipe generates no +masks (PR #847). The instrument flag image ships with the exposure and is split +per CCD by ``exp_split``; the sky-fixed healsparse masks are queried per object +inside the ShapePipe configs (``mask_query`` on exposures, ``make_cat`` on +tiles). Nothing is fetched, staged or rasterized, so there is nothing to +prepare. """ # No NUMBER_LIST for get_images — a download stage has nothing on disk to diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 74d985e2a..981ac8d65 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -41,18 +41,14 @@ The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor run that the shape chain does not need co-scheduled, and folding it in would add its runtime to a sum that has no room. -Note there is no `tile_mask` rule: the committed config chain is the -"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask -run), and no tile-mask config was committed in the S2 sweep. Adding the masked -variant is a config + one rule, at the config selector the PRD describes. - -That rule also needs a tile-side analogue of ``exp_star_cat``: tile star cats key -on TILE id, so they are a separate cache namespace and a separate node, and the -earliest point it can run is after ``tile_uncompress`` (create_star_cat.py's -``-k tile`` mode reads the uncompressed tile image's primary header). The mask -config would then read a real per-unit ``$SP_RUN/star_cat_tiles`` directory, -built the same way and for the same reason (the file handler intersects numbers -across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). +There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe +generates no masks: tiles have no instrument flag image of their own, so +tile_detect runs SExtractor with FLAG_IMAGE = False against +default_noimaflags.param (config_tile_Sx.ini — what used to be the "sx_nomask" +variant, now the only one). Sky-fixed masks reach the tile as CATALOGUE columns +instead: ``tile_make_cat``'s make_cat queries the configured healsparse maps at +every object's (RA, Dec) and writes one ``MASK_`` column per band, which +is what downstream selections cut on. """ # --- the node-local tile root (the I/O + scratch fix) ---------------------- @@ -176,8 +172,8 @@ def tile_local(tile): runs in `tile_vignets` and in each of the eight `tile_ngmix` chunks (not in `tile_merge_cats`, which has no pre_run), so the staging is attempted nine times per tile, eight of them concurrent siblings in one toposort level. It - is copy-to-a-temp-name plus `mv -f` -- the same all-or-nothing publish - shapepipe.utilities.file_io.write_atomic argues -- and it is UNCONDITIONAL, + is copy-to-a-temp-name plus `mv -f` -- an all-or-nothing publish, so a + reader never sees a partial file -- and it is UNCONDITIONAL, no `cp -u` and no already-there test. An interrupted `cp` leaves a truncated destination whose mtime is NEWER than the source, so `-u` would skip it forever, and nothing downstream would catch it: TILE_VIGNET_FRESH and @@ -390,9 +386,8 @@ def exp_manifests(wc, stage): return [ancient(p) for p in paths] def tile_exp_split(wc): return exp_manifests(wc, "exp_split") -def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") -def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_psf(wc) # Build the per-tile symlink forest. Declaring the exposure manifests as input @@ -873,10 +868,9 @@ rule tile_make_cat: # safe because nothing upstream of it survives, and tile_detect.json in # particular must never join the list. # -# A LOCALRULE (declared in the Snakefile), same as clean_exposure. Unlike -# exp_star_cat it is a DAG LEAF, so being local can never make it both a -# dependency and a dependent of a group — no `group:` label here, and none -# possible. +# A LOCALRULE (declared in the Snakefile), same as clean_exposure. It is a DAG +# LEAF, so being local can never make it both a dependency and a dependent of a +# group — no `group:` label here, and none possible. # # WHAT THIS SHARPENS ELSEWHERE: the TILE_LOCAL warning above says an edit to # tile_local() mid-campaign reruns finished tiles unsatisfiably because their diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 728f6c527..b763a5573 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,17 +7,12 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its -star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are -reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted -here like every other, so the exposure's chain must read as unbuilt, and 40 -symlinks left behind are a farm no rule now owns. The catalogue itself lives in -the run-independent cache, so rebuilding the farm costs a relink and no query. +run_sp_exp_Gie/Sp/SxSePsfPi), its ``manifests/`` and its ``logs/``. That is the +entire exposure store: since PR #847 removed ShapePipe's mask generation there +is no run_sp_exp_Ma tree and no star-catalogue link farm to reclaim beside it. Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not -``rmtree``d. Legacy unit dirs carry ``star_cat_exp`` as a link into the old -shared pool, and an rmtree would recurse through it and delete the shared cache -for every other exposure in the campaign. +``rmtree``d, so a link into a shared store can never be recursed through. Deleting the manifests is deliberate and load-bearing, not tidiness: @@ -98,10 +93,9 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - # is_symlink() first, and OR'd with exists(): exists() follows the link, so a - # dangling legacy star_cat_exp would otherwise be skipped and survive. - candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs", - args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") + # is_symlink() first, and OR'd with exists(): exists() follows the link, so + # a dangling link would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs") targets = [t for t in candidates if t.is_symlink() or t.exists()] # Tombstone first, complete — then delete (see the module docstring). diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index cf67363db..0dfac0ebb 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -14,8 +14,8 @@ ``shapepipe_run`` failed:: rc=0 - shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} --log {log} --job-rc "$rc" || rc=1 + shapepipe_run -c $SP_CONFIG/config_exp_Sp.ini -b {threads} || rc=$? + completeness.py check exp_split {output} --log {log} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory @@ -82,12 +82,18 @@ # --- exposure chain --- "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, - "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree # AND the bash baseline (both 120/exposure). + # + # mask_query is one sexcat_ext per CCD — the count the deleted exp_mask + # stage used to carry, now inside this chain because querying a healsparse + # map at ~2k detections needs no rule of its own. Its floor tracks the + # runners either side of it: setools tolerates sparse-CCD attrition, so a + # hard 40 here would fail exposures the chain is designed to survive. "exp_psf": { "sextractor_runner": dict(expect=120, floor=2), + "mask_query_runner": dict(expect=40, floor=2), "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), "psfex_runner": dict(expect=80, floor=2), "psfex_interp_runner": dict(expect=40, floor=0, warn=True), @@ -163,18 +169,17 @@ def check_floor(stage, run_dir): # var its config does, so chunk K's check looks at chunk K's dir. # # EVERY ENTRY HERE (and in COMPLETENESS above) HAS A RULE. The table used to -# carry two stages that did not: `tile_mask` (run_sp_tile_Ma, mask_runner 1/1) -# and `tile_detect_uc` (run_sp_tile_Uc). The committed config chain is the -# "sx_nomask" tile_detect variant and no tile-mask config was committed, so both -# were unreachable — tile.smk's docstring is where the masked variant is argued, -# and it is a config plus a rule plus these two rows, added back together. +# carry stages that did not — `tile_mask` (run_sp_tile_Ma) and `tile_detect_uc` +# (run_sp_tile_Uc) — and, until PR #847, an `exp_mask` stage that did. ShapePipe +# now generates no masks at all: the sky-fixed healsparse maps are queried per +# object inside the exp_psf and tile_make_cat chains, so masking has no stage +# of its own on either side and is not coming back. STAGE_DIR = { "tile_get_images": ("tile", "run_sp_tile_Git"), "tile_uncompress": ("tile", "run_sp_tile_Uz"), "tile_find_exposures": ("tile", "run_sp_tile_Fe"), "exp_get_images": ("exp", "run_sp_exp_Gie"), "exp_split": ("exp", "run_sp_exp_Sp"), - "exp_mask": ("exp", "run_sp_exp_Ma"), "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), "tile_detect": ("tile", "run_sp_tile_Sx"), diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index f559e3276..bd117c09c 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -59,7 +59,7 @@ TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", "tile_merge_headers", "tile_detect", "tile_vignets", "tile_ngmix", "tile_merge_cats", "tile_make_cat"] -EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] # The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names # the mechanism that owns each). Their presence is therefore NOT evidence that a diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py deleted file mode 100644 index 2407af71b..000000000 --- a/workflow/scripts/star_cats.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -"""The campaign's GSC 2.3 star catalogue, as a HEALPix-chunked sky store. - -Masking needs, for every exposure, the bright stars over its focal plane. The -sky does not change between exposures, so the network cost of that is a property -of the campaign's SKY AREA, not of its exposure count: exposures overlap each -other ~7-10 deep, and a tile's exposures all look at the same square degree. - -So the store is chunked by sky, not by exposure. One GSC 2.3 cone query per -HEALPix pixel of NSIDE=32, written run-independently under the ``star_cats`` -config root and never fetched twice. A campaign that grows past the fetched -footprint queries only the chunks its new tiles add; one that grows within it -queries nothing. - -Two numbers set the scale. A full-UNIONS footprint is ~1.5k chunks against ~25k -exposures, so the QUERY COUNT drops ~16x. The queried AREA drops ~4x: the old -design covered the footprint ~8 times over (that is just the exposure overlap -depth), the new one ~2 times, the 2x being the price of bounding a HEALPix -quadrilateral by the cone Vizier speaks (see ``pixel_cone``) — 5.6-9 deg^2 for a -3.36 deg^2 pixel, ~40-60k rows and ~3-4 MB per chunk. - -Two subcommands, one module, deliberately: ``fetch`` and ``cut`` must agree -EXACTLY on which pixel holds which star, and a shared NSIDE constant in one file -is the only version of that agreement which cannot drift. - - fetch --tile-list ... --store ... --manifest ... - The campaign side. Turns the tile list into the set of pixels its - exposures can possibly need, fetches the missing ones, writes a manifest. - - cut --images ... --store ... --out ... - The per-exposure side, purely local: read the focal-plane footprint from - the exposure's image headers, load the chunks covering it, deduplicate, - and cut to the focal-plane disc. Reproduces byte-for-byte the same sky - selection the old one-query-per-exposure cone did. - -Geometry, and why the fetch pad is what it is. Chunk-need is computed from the -TILE list rather than from exposure pointings, because tile IDs are the one thing -known before any download: a pointing center means reading a FITS header of an -image get_images has not fetched yet, and the DAG needs the chunk set at parse -time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so -each tile is a disc of half-diagonal 0.354 deg; find_exposures gives a tile every -exposure whose footprint covers it, and the MegaCam focal plane is a disc of -radius 0.73 deg (measured on the cached catalogues). An exposure center is -therefore at most 0.354 + 0.73 deg from the tile center, and its stars 0.73 deg -beyond that: 1.81 deg, padded to ``PAD_DEG`` = 2.0. The pad is a perimeter cost -— negligible for a contiguous campaign, and paid once. - -The pad is a bound, not a promise: ``cut`` verifies that every chunk covering the -exposure it was handed is on disk, and fails loudly if one is not. A missing -chunk means the geometry above is wrong, and that must not degrade quietly into -an under-masked exposure. -""" - -import argparse -import json -import sys -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -import numpy as np -import healpy as hp -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.table import Table, vstack - -# The PYTHONPATH pin in profiles/nibi puts this checkout's src/ on the path (see -# exposure.smk's in_container), the same way the vizier helper is reached below. -from shapepipe.utilities.file_io import write_atomic -from shapepipe.utilities.focal_plane import focal_plane_disc - -# GSC 2.3. The same catalogue the mask module's own CDS path uses -# (mask.py: _CDS_cat_ID), so the store is a drop-in for it. -CAT_ID = "I/305/out" - -# NSIDE=32 -> 3.36 deg^2 per pixel, 12288 pixels over the sky. Chosen so one -# chunk is a couple of MegaCam focal planes: small enough that a Vizier query -# stays within a small multiple of the per-exposure queries this replaces, large -# enough that a full-UNIONS footprint is ~1.5k chunks rather than ~25k. The -# cone-vs-quadrilateral overhead is scale-free, so NSIDE trades query count -# against query size and nothing else. NESTED, so a chunk id is a hierarchical -# sky address and a future NSIDE change is a subdivision. -NSIDE = 32 -NEST = True - -# Angular padding on the disc used to select chunks (see the module docstring). -PAD_DEG = 2.0 - -# The MegaCam focal-plane disc, and the margin added to a pixel's own bounding -# cone. Both in degrees. -MARGIN_DEG = 0.02 - -# GSC 2.3's object id: the deduplication key where chunk cones overlap. -ID_COL = "GSC2.3" - - -# --- the chunk store -------------------------------------------------------- - - -def store_dir(store: Path) -> Path: - """Chunks live under the catalogue and resolution that produced them, so a - later NSIDE or catalogue change is a new directory beside the old one rather - than a silent reinterpretation of files already on disk.""" - return Path(store) / CAT_ID.replace("/", "_") / f"nside{NSIDE}" - - -def chunk_path(store: Path, ipix: int) -> Path: - return store_dir(store) / f"star_chunk-{ipix:06d}.fits" - - -def chunks_for_disc(ra_deg: float, dec_deg: float, radius_deg: float) -> list[int]: - """Every pixel that touches the disc, as sorted ids. - - ``inclusive=True`` makes this a conservative superset — the guarantee ``cut`` - relies on is that no star inside the disc lives in a pixel this omits. - """ - vec = hp.ang2vec(ra_deg, dec_deg, lonlat=True) - return sorted(int(i) for i in hp.query_disc( - NSIDE, vec, np.radians(radius_deg), inclusive=True, fact=4, nest=NEST)) - - -def pixel_cone(ipix: int) -> tuple[float, float, float]: - """(ra, dec, radius_arcmin) of a cone that CONTAINS pixel ``ipix``. - - Vizier speaks cones, HEALPix speaks quadrilaterals, so the query is the - pixel's bounding cone: its center, and the largest center-to-boundary - distance plus a margin. The cone spills over the pixel edges, which costs a - little duplication between neighbours and buys the containment ``cut`` - depends on. The duplicates are removed on read, by ``ID_COL``. - """ - ra_c, dec_c = hp.pix2ang(NSIDE, ipix, nest=NEST, lonlat=True) - ra_b, dec_b = hp.vec2ang(hp.boundaries(NSIDE, ipix, step=8, nest=NEST).T, - lonlat=True) - center = SkyCoord(ra_c * u.deg, dec_c * u.deg) - radius = center.separation(SkyCoord(ra_b * u.deg, dec_b * u.deg)).deg.max() - return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) - - -def read_chunks(store: Path, ipixels: list[int]) -> Table: - """Load and deduplicate the given chunks. - - A missing chunk is fatal (see the module docstring): it means the fetch - footprint did not cover this exposure, and an under-masked exposure is worse - than a failed job. - """ - missing = [i for i in ipixels if not chunk_path(store, i).exists()] - if missing: - raise SystemExit( - f"star chunk(s) {missing} not in {store_dir(store)}. The campaign's " - f"star_catalogue fetch did not cover this exposure — re-run it " - f"(and check that its tile list contains this exposure's tiles).") - - table = vstack([Table.read(chunk_path(store, i)) for i in ipixels], - metadata_conflicts="silent") - _, keep = np.unique(np.asarray(table[ID_COL]), return_index=True) - return table[np.sort(keep)] - - -# --- exposure footprint ----------------------------------------------------- -# The WCS construction and the focal-plane disc live in -# shapepipe.utilities.focal_plane, beside the vizier helper and imported the same -# way: create_star_cat.py needs exactly the same geometry, and the two must not -# be able to disagree about which sky an exposure covers. - - -def exposure_image(images_dir: Path) -> Path: - """The one multi-extension exposure image in a get_images output dir. - - That dir is a symlink farm holding ``image-.fitsfz`` plus its weight and - flag; only the image carries the 40 CCD WCSs. - """ - found = sorted(p for p in Path(images_dir).iterdir() if "image" in p.name) - if not found: - raise SystemExit(f"no image file in {images_dir}") - return found[0] - - -# --- fetch ------------------------------------------------------------------ - - -def campaign_chunks(tile_ids: list[str]) -> list[int]: - """Every chunk the campaign's exposures can need, from the tile list alone.""" - from shapepipe.utilities.cfis import get_tile_coord_from_nixy - - needed: set[int] = set() - for tile_id in tile_ids: - nix, niy = tile_id.split(".") - ra, dec = get_tile_coord_from_nixy(nix, niy) - needed.update(chunks_for_disc(ra.degree, dec.degree, PAD_DEG)) - return sorted(needed) - - -def fetch(args: argparse.Namespace) -> None: - from shapepipe.utilities.vizier import query_vizier - - tile_ids = [ln.strip() for ln in Path(args.tile_list).read_text().splitlines() - if ln.strip()] - needed = campaign_chunks(tile_ids) - out_dir = store_dir(args.store) - out_dir.mkdir(parents=True, exist_ok=True) - todo = [i for i in needed if not chunk_path(args.store, i).exists()] - print(f"star chunks: {len(needed)} needed for {len(tile_ids)} tiles, " - f"{len(todo)} to fetch -> {out_dir}", file=sys.stderr) - - def one(ipix: int) -> int: - ra, dec, radius_arcmin = pixel_cone(ipix) - table = query_vizier(ra, dec, radius_arcmin, CAT_ID) - write_atomic(table, chunk_path(args.store, ipix)) - print(f"chunk {ipix}: {len(table)} rows " - f"(ra={ra:.4f} dec={dec:.4f} r={radius_arcmin:.1f}')", - file=sys.stderr) - return len(table) - - # A handful of concurrent queries, never one per exposure: the same modest - # concurrency the per-exposure rule reached through --local-cores, now an - # explicit number instead of an accident of the head node's CPU count. - if todo: - with ThreadPoolExecutor(max_workers=args.workers) as pool: - list(pool.map(one, todo)) - - write_manifest(args, tile_ids, needed, len(todo)) - - -def write_manifest(args, tile_ids, needed, n_fetched) -> None: - """The rule's declared output. - - Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and - has no per-runner count floors, so there is nothing to compose and no - separate ``log:`` — under ``set -euo pipefail`` the job either completes or - aborts at the failing query, and snakemake's captured stderr is the evidence. - The manifest keeps the workflow's "one rule, one manifest" currency: written - last, and only when the content changed, so an unchanged campaign leaves the - mtime where it was rather than churning the `mtime` rerun-trigger. - """ - body = json.dumps({ - "stage": "star_catalogue", - "level": "campaign", - "status": "complete", - "catalogue": CAT_ID, - "nside": NSIDE, - "nest": NEST, - "pad_deg": PAD_DEG, - "store": str(store_dir(args.store)), - "n_tiles": len(tile_ids), - "n_chunks": len(needed), - "n_fetched": n_fetched, - "chunks": needed, - }, indent=2, sort_keys=True) - path = Path(args.manifest) - path.parent.mkdir(parents=True, exist_ok=True) - if not path.exists() or path.read_text() != body: - path.write_text(body) - - -# --- cut -------------------------------------------------------------------- - - -def cut(args: argparse.Namespace) -> None: - image = exposure_image(args.images) - ra, dec, radius = focal_plane_disc(image) - ipixels = chunks_for_disc(ra, dec, radius) - table = read_chunks(args.store, ipixels) - - center = SkyCoord(ra * u.deg, dec * u.deg) - stars = SkyCoord(np.asarray(table["RAJ2000"]) * u.deg, - np.asarray(table["DEJ2000"]) * u.deg) - inside = table[center.separation(stars).deg <= radius] - - print(f"{image.name}: ra={ra:.4f} dec={dec:.4f} r={radius:.4f} deg, " - f"{len(ipixels)} chunks -> {len(inside)} stars", file=sys.stderr) - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - write_atomic(inside, out) - - -# --- CLI -------------------------------------------------------------------- - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - sub = p.add_subparsers(dest="cmd", required=True) - - f = sub.add_parser("fetch", help="fetch the campaign footprint's chunks") - f.add_argument("--tile-list", required=True, type=Path) - f.add_argument("--store", required=True, type=Path) - f.add_argument("--manifest", required=True, type=Path) - f.add_argument("--workers", type=int, default=4) - f.set_defaults(func=fetch) - - c = sub.add_parser("cut", help="cut one exposure's catalogue from the store") - c.add_argument("--images", required=True, type=Path, - help="a get_images output dir holding image-.fitsfz") - c.add_argument("--store", required=True, type=Path) - c.add_argument("--out", required=True, type=Path) - c.set_defaults(func=cut) - - args = p.parse_args() - args.func(args) - - -if __name__ == "__main__": - main()