From 28b0b929e76a29533e00457b8cc813230482744b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:07:02 +0200 Subject: [PATCH 01/44] test(ngmix): specify defect fill, epoch cuts and shear recovery near defects Defects (nonzero flag, zero weight, invalid RMS) must be zero-weighted and noise-filled under every BLEND_HANDLING, with neighbour pixels left raw under uberseg. The masked-fraction cut must count that same raw set, and an epoch with a defect closer than EPOCH_CENTRAL_DEFECT_RADIUS to the stamp centre must be dropped, at a strict boundary and at the configured radius. The runner must pass both cut options on, the HSM centroid must ignore raw defect values, each epoch must keep its own OFFSET, and every tile must log its epoch-cut tally. The science test recovers the full 2x2 response for columns, 3-px bleeds, single pixels and edge bands at the veto radius, on 0.3" and 0.5" galaxies through round and elliptical 0.7" PSFs, and requires |c1|, |c2| < 5e-4 and |m11|, |m22| < 1%. Flagged pixels in the simulation hold a hot value, so an unfilled defect shows up. A column two pixels inside the radius is the positive control. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- tests/helpers/defect_response.py | 89 +++++ tests/module/test_ngmix_defect_fill.py | 532 +++++++++++++++++++++++++ tests/module/test_ngmix_uberseg.py | 69 ++-- tests/science/test_defect_veto.py | 104 +++++ 4 files changed, 755 insertions(+), 39 deletions(-) create mode 100644 tests/helpers/defect_response.py create mode 100644 tests/module/test_ngmix_defect_fill.py create mode 100644 tests/science/test_defect_veto.py diff --git a/tests/helpers/defect_response.py b/tests/helpers/defect_response.py new file mode 100644 index 000000000..c3e3ac157 --- /dev/null +++ b/tests/helpers/defect_response.py @@ -0,0 +1,89 @@ +"""Full-matrix metacal recovery for fixed detector defects.""" + +import numpy as np + +from shapepipe.modules.ngmix_package import ngmix as ngm +from shapepipe.testing.simulate import make_data +from tests.helpers.metacal_sim import build_stamp + + +def defect_response(bad, hlr=0.5, psf=0.7, seeds=range(4), + psf_shear=(0.0, 0.0), options=None, known_rms=True, + defect_value=1e3): + """Recover c and M = inverse(mean R) A - I with paired seed errors. + + Null pairs rotate the pixels by 90 degrees while pre-rotating the PSF + ellipticity oppositely, so the final PSF stays fixed in detector + coordinates. + This does NOT average away elliptical-PSF leakage. + + The flagged pixels ``bad`` hold ``defect_value`` (far above the galaxy's + peak) rather than sky, as a hot column or bleed would, so any defect value + that reaches metacal shows up as a bias. + """ + gamma = 0.02 + options = {} if options is None else options + samples = [] + for seed in seeds: + def arm(axis, sign, rotation=0): + shear = [0.0, 0.0] + if axis >= 0: + shear[axis] = sign * gamma + ps = tuple(v * (-1 if rotation else 1) for v in psf_shear) + data = list(make_data( + rng=np.random.RandomState(seed + 100), shear=shear, + psf_shear=ps, noise=1e-4, n_epochs=1, img_size=51, + gal_hlr=hlr, psf_fwhm=psf, return_centers=True, + )) + centre = data.pop()[0] + offset = np.array([centre.y - 26, centre.x - 26]) + if rotation: + data[0] = [np.rot90(a).copy() for a in data[0]] + data[1] = [np.rot90(a).copy() for a in data[1]] + offset = np.array([-offset[1], offset[0]]) + data[0] = [np.where(bad, defect_value, a) for a in data[0]] + data[4] = [bad.astype(np.int32)] + stamp = build_stamp(data) + stamp.offsets = [offset] + if known_rms: + stamp.bkg_rms = [np.full((51, 51), 1e-4)] + rng = np.random.RandomState(seed) + result, _, _ = ngm.do_ngmix_metacal( + stamp, ngm.get_prior(0.1857, rng), 1.0, rng, + centroid_source="wcs", **options, + ) + assert all(result[t]["flags"] == 0 for t in ngm.METACAL_TYPES) + e = np.asarray(result["noshear"]["g"]) + response = np.column_stack([ + (np.asarray(result[p]["g"]) - result[m]["g"]) / 0.02 + for p, m in (("1p", "1m"), ("2p", "2m")) + ]) + assert np.all(np.isfinite(e)) and np.all(np.isfinite(response)) + return e, response + null = [arm(-1, 0, k) for k in (0, 1)] + e0 = np.mean([a[0] for a in null], axis=0) + r0 = np.mean([a[1] for a in null], axis=0) + derivatives, responses = [], [] + for axis in (0, 1): + plus, minus = arm(axis, 1), arm(axis, -1) + derivatives.append((plus[0] - minus[0]) / (2 * gamma)) + responses.extend([plus[1], minus[1]]) + samples.append((e0, r0, np.column_stack(derivatives), + np.mean(responses, axis=0))) + e, r, a, rm = [np.array([s[i] for s in samples]) for i in range(4)] + assert np.linalg.svd(rm.mean(axis=0), compute_uv=False).min() > 0.1 + c = np.linalg.solve(r.mean(axis=0), e.mean(axis=0)) + matrix = np.linalg.solve(rm.mean(axis=0), a.mean(axis=0)) - np.eye(2) + draw = np.random.RandomState(91).randint(len(e), size=(1000, len(e))) + cb = np.linalg.solve(r[draw].mean(axis=1), + e[draw].mean(axis=1)[..., None])[..., 0] + mb = np.linalg.solve(rm[draw].mean(axis=1), a[draw].mean(axis=1)) + mb -= np.eye(2) + return dict(c=c.tolist(), m=np.diag(matrix).tolist(), + matrix=matrix.tolist(), + c_err=cb.std(axis=0).tolist(), + m_err=np.diag(mb.std(axis=0)).tolist(), + response=r.mean(axis=0).tolist(), + seed_groups=[dict(e=s[0].tolist(), R=s[1].tolist(), + A=s[2].tolist(), Rm=s[3].tolist()) + for s in samples]) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py new file mode 100644 index 000000000..0480c7ef7 --- /dev/null +++ b/tests/module/test_ngmix_defect_fill.py @@ -0,0 +1,532 @@ +"""Defect fill and the epoch cuts (ngmix module). + +A defect is a stamp pixel with a nonzero flag, zero exposure weight or an +invalid background RMS. Before metacal, :func:`prepare_ngmix_weights` gives +every defect weight 0 and fills it with noise at its background RMS, whatever +``BLEND_HANDLING`` is. Under uberseg, pixels on the neighbour side only lose +their weight, and their image values stay raw. The per-epoch cuts in +:func:`prepare_postage_stamps` act on the same defect set: the masked-fraction +cut counts it, and the central-defect veto drops an epoch with a defect near +the stamp centre. +""" + +import re +from types import SimpleNamespace + +import numpy as np +import numpy.testing as npt +from astropy.io import fits +from astropy.wcs import WCS +from hypothesis import given +from hypothesis import strategies as st +from sqlitedict import SqliteDict +from shapepipe.modules.ngmix_package import ngmix as ngmix_module + +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + Ngmix, + make_ngmix_observation, + prepare_ngmix_weights, + prepare_postage_stamps, + uberseg_weight, +) + + +# --- prepare_ngmix_weights: the filled set is the defect set --------------- + +@st.composite +def defect_stamps(draw): + """Square stamp with random flagged, zero-weight and bad-RMS pixels.""" + n = draw(st.integers(min_value=5, max_value=21)) + pixels = st.lists( + st.tuples(st.integers(0, n - 1), st.integers(0, n - 1)), max_size=n + ) + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + bkg_rms = np.ones((n, n)) + for i, j in draw(pixels): + weight[i, j] = 0.0 + for i, j in draw(pixels): + flag[i, j] = draw(st.sampled_from([1, 2, 2**10])) + for i, j in draw(pixels): + bkg_rms[i, j] = draw(st.sampled_from([0.0, -1.0, np.nan, np.inf])) + # Raw values far outside the unit-RMS noise, so a filled pixel is + # unambiguous. + gal = 1.0e3 + np.arange(n * n, dtype=float).reshape(n, n) + return gal, weight, flag, bkg_rms + + +def _uberseg_seg(n): + """Central object on the centre pixel, neighbour footprint in a corner.""" + seg = np.zeros((n, n), dtype=np.int32) + seg[n // 2, n // 2] = 1 + seg[:2, :2] = 2 + return seg + + +@given( + stamp=defect_stamps(), + blend_handling=st.sampled_from(["none", "uberseg"]), + seed=st.integers(0, 2**31 - 1), +) +def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): + """Filled pixels are exactly the defects (flag, zero weight, bad RMS). + They carry zero weight and look like noise. Every other pixel keeps its + raw value, under either BLEND_HANDLING. + + Failure modes: + * a defect source (flag, zero weight, bad RMS) is left out of the fill; + * the filled set grows beyond the defects (for example symmetrized); + * the filled set and the zero-weight defect set differ; + * the fill is skipped under uberseg; + * neighbour-side pixels are filled. + """ + gal, weight, flag, bkg_rms = stamp + n = gal.shape[0] + kwargs = ( + dict(seg=_uberseg_seg(n), object_number=1, dilate_neighbour=1) + if blend_handling == "uberseg" + else {} + ) + + gal_out, w_out, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(seed), bkg_rms=bkg_rms, + blend_handling=blend_handling, **kwargs, + ) + + defect = ( + (weight == 0) | (flag != 0) | ~(np.isfinite(bkg_rms) & (bkg_rms > 0)) + ) + if defect.all(): + return # fully masked: no clean pixel left to set the noise level + filled = gal_out != gal + npt.assert_array_equal(filled, defect, "filled set is not the defect set") + assert np.all(np.abs(gal_out[filled]) < 10.0), "fill is not unit noise" + neighbour = ( + uberseg_weight(np.ones((n, n)), kwargs["seg"], 1, dilate_neighbour=1) + == 0.0 + if blend_handling == "uberseg" + else np.zeros((n, n), dtype=bool) + ) + # Zero weight on defects and neighbour side; neighbour pixels that are + # not defects keep their raw values (filled-set equality above). + npt.assert_array_equal(w_out == 0.0, defect | neighbour) + npt.assert_array_equal(w_out[~(defect | neighbour)], 1.0) + + +def test_uberseg_defect_in_neighbour_region_is_filled(): + """A defect pixel that also lies on the neighbour side is filled. + + Failure mode: the fill is restricted to pixels uberseg keeps, so a raw bad + pixel on the neighbour side still reaches metacal. + """ + n = 21 + gal = 1.0e3 + np.arange(n * n, dtype=float).reshape(n, n) + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + flag[1, 1] = 1 # inside the neighbour footprint + gal_out, w_out, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), bkg_rms=np.ones((n, n)), + blend_handling="uberseg", seg=_uberseg_seg(n), object_number=1, + ) + assert w_out[1, 1] == 0.0 and gal_out[1, 1] != gal[1, 1] + # A neighbour-side pixel that is not a defect: zero weight, raw value. + assert w_out[0, 3] == 0.0 and gal_out[0, 3] == gal[0, 3] + + +# --- prepare_postage_stamps: the fraction cut counts the defect set -------- + +N_STAMP = 51 +RA, DEC = 150.0, 2.0 + + +def _fake_inputs(epochs): + """Minimal vignet / tile-catalogue stand-ins for prepare_postage_stamps. + + ``epochs`` maps ``"-"`` to ``(flag, weight)`` or + ``(flag, weight, bkg_rms)``. Returns ``(vignet, tile_cat, psf_obj, + gal_obj)``. + """ + rng = np.random.default_rng(1) + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + wcs.wcs.crval = [RA, DEC] + wcs.wcs.crpix = [N_STAMP / 2, N_STAMP / 2] + wcs.wcs.cdelt = [-0.187 / 3600, 0.187 / 3600] + header = fits.Header({"FSCALE": 1.0}).tostring() + + def per_epoch(make): + return {k: {"VIGNET": make(k)} for k in epochs} + + with_rms = any(len(v) == 3 for v in epochs.values()) + psf_obj = per_epoch(lambda k: np.ones((N_STAMP, N_STAMP))) + gal_obj = per_epoch(lambda k: rng.normal(0.0, 1.0, (N_STAMP, N_STAMP))) + vignet = SimpleNamespace( + gal_vign_cat={"1": gal_obj}, + bkg_vign_cat=None, + bkg_rms_vign_cat=( + {"1": per_epoch( + lambda k: epochs[k][2] if len(epochs[k]) == 3 + else np.ones((N_STAMP, N_STAMP)) + )} + if with_rms + else None + ), + flag_vign_cat={"1": per_epoch(lambda k: epochs[k][0])}, + weight_vign_cat={"1": per_epoch(lambda k: epochs[k][1])}, + f_wcs_file={ + k.split("-")[0]: { + int(k.split("-")[1]): {"WCS": wcs, "header": header} + } + for k in epochs + }, + ) + tile_cat = SimpleNamespace( + vign=None, seg=None, ra=np.array([RA]), dec=np.array([DEC]) + ) + return vignet, tile_cat, psf_obj, gal_obj + + +def _two_sided_band(width): + """Mask of ``width`` columns on each side of the stamp, far from the + centre (at least 17 px for width 9).""" + band = np.zeros((N_STAMP, N_STAMP), dtype=bool) + band[:, :width] = True + band[:, -width:] = True + return band + + +def _epochs(): + """A clean epoch and four masked ones, all defects far from the centre. + + * band10: 10 flagged columns on one side, 19.6% (39% if symmetrized); + * flag18: 2 x 9 flagged columns, 35.3%; + * dead18: the same columns at zero weight, no flags; + * rms18: the same columns with an invalid background RMS. + """ + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + band10 = clean.copy() + band10[:, -10:] = 1 + two = _two_sided_band(9) + dead = ones.copy() + dead[two] = 0.0 + rms = ones.copy() + rms[two] = np.nan + return { + "2100001-10": (clean, ones), + "2100002-11": (band10, ones), + "2100003-12": (two.astype(np.int32), ones), + "2100004-13": (clean.copy(), dead), + "2100005-14": (clean.copy(), ones, rms), + } + + +def _surviving(epochs, **kwargs): + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(epochs) + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, **kwargs, + ) + names = {id(v[0]): name for name, v in epochs.items()} + return sorted(names[id(flag)] for flag in stamp.flags) + + +def test_epoch_cut_counts_the_defect_set(): + """At the default 1/3 cut, the three 35% epochs are dropped, whichever + defect source masks them, and the 20% band survives. + + Failure modes: the cut counts flags only (keeps dead18 and rms18), omits + one defect source, or counts a symmetrized set (drops band10); in each + case the cut disagrees with the set that is zero-weighted and filled + (epoch-cut-on-defect-mask). + """ + assert _surviving(_epochs()) == ["2100001-10", "2100002-11"] + + +def test_epoch_cut_threshold_is_the_configured_fraction(): + """At a 10% cut (the DES Y3 and Y6 value), the 19.6% band is dropped as + well, and only the clean epoch survives. + + Failure mode: the configured threshold is ignored. + """ + assert _surviving(_epochs(), epoch_masked_fraction_cut=0.1) == [ + "2100001-10" + ] + + +# --- prepare_postage_stamps: the central-defect veto ----------------------- + +def _veto_epochs(radius): + """A clean epoch, one with a single flagged pixel just inside + ``radius`` of the stamp centre, and one with a flagged column exactly + ``radius`` away. Both masked epochs are far below the fraction cut. + """ + centre = N_STAMP // 2 + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + near, far = clean.copy(), clean.copy() + near[centre, centre + radius - 1] = 1 + far[:, centre + radius] = 1 + return { + "2100001-10": (clean, ones), + "2100002-11": (near, ones), + "2100003-12": (far, ones), + } + + +def test_central_defect_vetoes_the_epoch(): + """At the default radius, a single defect pixel inside it drops the + epoch, and a column at the radius does not. + + Failure mode: the veto is skipped, so an epoch whose filled hole overlaps + the object's light enters the fit; or the boundary is inclusive, dropping + the column at the radius (epoch-central-defect-veto). + """ + epochs = _veto_epochs(EPOCH_CENTRAL_DEFECT_RADIUS) + assert _surviving(epochs) == ["2100001-10", "2100003-12"] + + +def test_central_defect_radius_is_the_configured_value(): + """Radius 0 disables the veto; a radius one pixel larger than the far + column's distance drops that epoch too. + + Failure mode: the configured radius is ignored. + """ + epochs = _veto_epochs(EPOCH_CENTRAL_DEFECT_RADIUS) + assert _surviving(epochs, epoch_central_defect_radius=0) == [ + "2100001-10", "2100002-11", "2100003-12" + ] + assert _surviving( + epochs, epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS + 1 + ) == ["2100001-10"] + + +# --- prepare_postage_stamps: per-epoch OFFSET ------------------------------ + +def test_each_surviving_epoch_carries_its_own_offset(): + """``stamp.offsets`` holds each surviving epoch's vignette OFFSET, in the + order of ``stamp.flags``. + + Failure mode: the offset is dropped or read from another epoch, so the + default "wcs" centroid raises or puts the Jacobian origin off the object. + """ + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + epochs = {f"210000{i}-1{i}": (clean.copy(), ones) for i in range(3)} + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(epochs) + del vignet.gal_vign_cat # OFFSET must come from the supplied gal_obj. + for i, name in enumerate(epochs): + gal_obj[name]["OFFSET"] = np.array([0.1 * i, -0.1 * i]) + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, + ) + names = {id(flag): name for name, (flag, _) in epochs.items()} + assert len(stamp.offsets) == len(stamp.flags) == 3 + for flag, offset in zip(stamp.flags, stamp.offsets): + npt.assert_array_equal(offset, gal_obj[names[id(flag)]]["OFFSET"]) + + +# --- Ngmix.process: the per-tile epoch-cut tally --------------------------- + +class _RecordingLogger: + def __init__(self): + self.messages = [] + + def info(self, msg, *_args, **_kwargs): + self.messages.append(msg) + + +def test_process_logs_the_epoch_cut_tally(tmp_path, monkeypatch): + """One tile, four objects; the end-of-tile line counts each cut's drops. + + * object 1: clean, 18-column edge band, defect inside the radius -> one + epoch each for considered, masked_fraction, central_veto; survives. + * object 2: edge band and central defect -> both epochs dropped; emptied. + * object 3: one all-zero stamp, skipped before the cuts -> not considered, + and not emptied by the cuts. + * object 4: no PSF ('empty') -> never reaches the cuts. + + Failure modes: a cut's drops are not counted or land in the wrong + counter; epochs skipped before the cuts are counted as considered; an + object with no epoch at all is reported as emptied by the cuts; counts + from one object overwrite another's. + """ + radius = EPOCH_CENTRAL_DEFECT_RADIUS + centre = N_STAMP // 2 + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + wide18, near = clean.copy(), clean.copy() + wide18[:, -18:] = 1 + near[centre, centre + radius - 1] = 1 + objects = { + 1: {"2100001-10": (clean, ones), "2100002-11": (wide18, ones), + "2100003-12": (near, ones)}, + 2: {"2100004-13": (wide18.copy(), ones), + "2100005-14": (near.copy(), ones)}, + 3: {"2100006-15": (clean.copy(), ones)}, + } + stores = {} + for obj_id, epochs in objects.items(): + vignet, _, psf_obj, gal_obj = _fake_inputs(epochs) + if obj_id == 3: + gal_obj["2100006-15"]["VIGNET"] = np.zeros((N_STAMP, N_STAMP)) + stores[obj_id] = (vignet, psf_obj, gal_obj) + vignet = SimpleNamespace( + bkg_vign_cat=None, + bkg_rms_vign_cat=None, + psf_vign_cat={ + "4": "empty", **{str(i): s[1] for i, s in stores.items()} + }, + gal_vign_cat={ + "4": "empty", **{str(i): s[2] for i, s in stores.items()} + }, + flag_vign_cat={ + str(i): s[0].flag_vign_cat["1"] for i, s in stores.items() + }, + weight_vign_cat={ + str(i): s[0].weight_vign_cat["1"] for i, s in stores.items() + }, + f_wcs_file={ + k: v for s in stores.values() for k, v in s[0].f_wcs_file.items() + }, + close=lambda: None, + ) + tile_cat = SimpleNamespace( + obj_id=np.array([1, 2, 3, 4]), ra=np.full(4, RA), dec=np.full(4, DEC), + vign=None, seg=None, flux=None, + ) + + paths = [tmp_path / f"{name}.sqlite" for name in + ("gal", "psf", "weight", "flag", "headers")] + for path in paths: + SqliteDict(str(path)).close() + log = _RecordingLogger() + ngmix = Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:4]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), log, + bkg_sub=False, + ) + ngmix._vignet_cat.close() + ngmix._vignet_cat = vignet + monkeypatch.setattr(ngmix_module, "Tile_cat", lambda *a, **k: tile_cat) + + def no_fit(*_args, **_kwargs): + raise RuntimeError("metacal is not under test") + + monkeypatch.setattr(ngmix_module, "do_ngmix_metacal", no_fit) + for method in ("compile_results", "save_results", "log_mean_ellipticity"): + monkeypatch.setattr(Ngmix, method, lambda *_a, **_k: None) + + ngmix.process() + + lines = [m for m in log.messages if m.startswith("epoch cuts:")] + assert len(lines) == 1, log.messages + tally = dict( + (k, int(v)) for k, v in re.findall(r"(\w+)=(\d+)", lines[0]) + ) + assert tally == dict( + considered=5, masked_fraction=2, central_veto=2, objects_emptied=1 + ), lines[0] + + +# --- ngmix_runner: the epoch-cut options reach Ngmix ------------------------ + +class _OptionConfig: + """Config stub answering from a dict; absent options take the caller's + fallback.""" + + def __init__(self, options): + self._options = {"MAG_ZP": "30.0", "ID_OBJ_MIN": "-1", + "ID_OBJ_MAX": "-1", **options} + + def has_option(self, _sec, key): + return key in self._options + + def get(self, _sec, key): + return self._options[key] + + def getexpanded(self, _sec, key): + return self._options[key] + + def getfloat(self, _sec, key): + return float(self._options[key]) + + def getint(self, _sec, key): + return int(self._options[key]) + + def getboolean(self, _sec, key, fallback=False): + return fallback + + +def test_runner_threads_the_epoch_cut_options(tmp_path, monkeypatch): + """EPOCH_CENTRAL_DEFECT_RADIUS and EPOCH_MASKED_FRACTION_CUT reach Ngmix + as configured, and default to the module constants when absent. + + Failure mode: the runner drops or ignores an option, so a configured A/B + arm silently runs the default cuts. + """ + from shapepipe.modules import ngmix_runner as runner_module + + captured = [] + + class _Capture: + def __init__(self, *_args, **kwargs): + captured.append(kwargs) + + def process(self): + pass + + monkeypatch.setattr(runner_module, "Ngmix", _Capture) + inputs = [str(tmp_path / f"in{i}.sqlite") for i in range(7)] + for path in inputs: + SqliteDict(path).close() + + for options, radius, fraction in ( + ({"EPOCH_CENTRAL_DEFECT_RADIUS": "7.5", + "EPOCH_MASKED_FRACTION_CUT": "0.1"}, 7.5, 0.1), + ({}, EPOCH_CENTRAL_DEFECT_RADIUS, + ngmix_module.EPOCH_MASKED_FRACTION_CUT), + ): + runner_module.ngmix_runner( + inputs, {"output": str(tmp_path)}, "-001-001", + _OptionConfig(options), "NGMIX_RUNNER", _RecordingLogger(), + ) + assert captured[-1]["epoch_central_defect_radius"] == radius + assert captured[-1]["epoch_masked_fraction_cut"] == fraction + + +# --- make_ngmix_observation: the HSM centroid reads the filled image ------- + +def test_hsm_centroid_ignores_raw_defect_values(): + """With ``centroid_source="hsm"``, the Jacobian origin lands on the + object even when a flagged column a few pixels away holds a raw value + ten times the object's peak. + + Failure mode: HSM measures the raw stamp, so the defect drags the centroid + off the object (or HSM fails and falls back to the stamp centre). + """ + import galsim + + n = 51 + centre = (n - 1) / 2 + d_row, d_col = 1.3, -0.7 + rows, cols = np.mgrid[:n, :n] + gal = np.exp( + -((rows - centre - d_row) ** 2 + (cols - centre - d_col) ** 2) + / (2 * 2.0 ** 2) + ) + flag = np.zeros((n, n), dtype=np.int32) + flag[:, n // 2 + 6] = 1 + gal[flag != 0] = 10.0 + psf = np.exp(-((rows - centre) ** 2 + (cols - centre) ** 2) / 2.0) + obs = make_ngmix_observation( + gal, np.ones((n, n)), flag, psf / psf.sum(), + galsim.PixelScale(0.1857).jacobian(), np.random.RandomState(0), + bkg_rms=np.full((n, n), 1e-3), centroid_source="hsm", + ) + row, col = obs.jacobian.get_cen() + npt.assert_allclose( + [row - centre, col - centre], [d_row, d_col], atol=0.05 + ) diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 0c80486b2..86dad33bc 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -6,10 +6,10 @@ Geometry assertions on a synthetic two-object stamp: neighbour-side pixels are zeroed, the surviving central core is a *single connected* region (the emergent "circularisation"), and the neighbour footprint is fully removed. -* :func:`prepare_ngmix_weights` — the ``noisefill`` default is byte-for-byte - unchanged (asserted against an independent recomputation of the legacy - three-line noise-fill on a shared RNG), while ``uberseg`` hard-masks the - weight (weight -> 0) and leaves the image untouched. +* :func:`prepare_ngmix_weights` under ``uberseg`` — neighbour-side pixels + lose their weight and keep their raw image values, while defect pixels are + noise-filled as under any blend handling (the defect fill itself is covered + in ``test_ngmix_defect_fill.py``). * The error contract when ``uberseg`` is selected without a segmentation map (the seg-map source is plumbing-gated upstream). """ @@ -226,7 +226,7 @@ def test_uberseg_matches_bruteforce_nearest_segment(): npt.assert_array_equal(out, brute) -# --- prepare_ngmix_weights: default unchanged, uberseg hard-masks ---------- +# --- prepare_ngmix_weights: uberseg zeroes neighbour weights only --------- def _gal_flag_weight(npix=41, seed=7): rng = np.random.default_rng(seed) @@ -239,32 +239,6 @@ def _gal_flag_weight(npix=41, seed=7): return gal, flag, weight -def test_noisefill_default_is_byte_identical_to_legacy(): - """The default path reproduces the legacy three-line noise-fill exactly - (same RNG stream): masked pixels replaced by noise, weight 1/sigma^2.""" - gal, flag, weight = _gal_flag_weight() - - gal_out, w_out, noise_out = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(123), - ) - - # Independent recomputation of the legacy algorithm on the same stream. - from modopt.math.stats import sigma_mad - rng = np.random.RandomState(123) - mask = np.copy(weight) != 0 - mask[flag != 0] = False - sig = sigma_mad(gal) - w_exp = mask.astype(float) / sig ** 2 - noise_exp = rng.standard_normal(gal.shape) * sig - noise_gal = rng.standard_normal(gal.shape) * sig - gal_exp = np.copy(gal) - gal_exp[~mask] = noise_gal[~mask] - - npt.assert_array_equal(gal_out, gal_exp) - npt.assert_array_equal(w_out, w_exp) - npt.assert_array_equal(noise_out, noise_exp) - - def test_noisefill_ignores_seg_and_dilate_kwargs(): """Under noisefill, passing seg / dilate_neighbour changes nothing: the result matches the plain default call on the same RNG stream.""" @@ -282,9 +256,15 @@ def test_noisefill_ignores_seg_and_dilate_kwargs(): npt.assert_array_equal(a, b) -def test_uberseg_hard_masks_weight_and_leaves_image_untouched(): - """uberseg: image returned untouched, weight zeroed on neighbour-side and - flagged pixels, positive on the central core.""" +def test_uberseg_fills_defects_and_leaves_neighbour_pixels_raw(): + """uberseg: flagged pixels are noise-filled at weight 0; neighbour-side + pixels get weight 0 and keep their raw image values; the central core + keeps weight and image. + + Failure modes: the defect fill is skipped under uberseg, so raw bad + pixels reach metacal; or the neighbour side is noise-filled + (defects-filled-neighbours-raw). + """ npix = 41 gal, flag, weight = _gal_flag_weight(npix=npix) seg, centre, neigh = two_object_seg(npix=npix, sep=12) @@ -294,13 +274,16 @@ def test_uberseg_hard_masks_weight_and_leaves_image_untouched(): blend_handling="uberseg", seg=seg, object_number=1, ) - # Image untouched under uberseg (no noise fill). - npt.assert_array_equal(gal_out, gal) - # Neighbour footprint hard-masked; central centre kept. + # Flagged pixels: zero weight, image replaced by noise. + for pix in [(5, 5), (30, 12)]: + assert w_out[pix] == 0.0 + assert gal_out[pix] != gal[pix] + # Neighbour footprint: zero weight, raw image (never noise-filled). assert np.all(w_out[seg == 2] == 0.0) + npt.assert_array_equal(gal_out[seg == 2], gal[seg == 2]) + # Central core: weight and image untouched. assert w_out[centre] > 0.0 - # Flagged bad pixels remain at weight 0 (folded into the base mask). - assert w_out[5, 5] == 0.0 + assert gal_out[centre] == gal[centre] def test_uberseg_requires_seg_and_object_number(): @@ -422,3 +405,11 @@ def test_runner_missing_seg_vignet_file_raises(tmp_path): "NGMIX_RUNNER", _RecordingLogger(), ) + + +def test_retired_noisefill_name_is_rejected(): + """Do not silently interpret the retired neighbour option.""" + with pytest.raises(ValueError, match="noisefill"): + prepare_ngmix_weights(np.ones((5, 5)), np.ones((5, 5)), + np.zeros((5, 5)), np.random.RandomState(3), + blend_handling="noisefill") diff --git a/tests/science/test_defect_veto.py b/tests/science/test_defect_veto.py new file mode 100644 index 000000000..3384263e4 --- /dev/null +++ b/tests/science/test_defect_veto.py @@ -0,0 +1,104 @@ +"""Shear recovery for the defects the central-defect veto keeps (noise fill). + +Physics invariant: a defect the veto keeps -- a column, a 3-px bleed or a +single pixel at the veto radius or beyond, or an edge band -- leaves both +additive terms |c1|, |c2| < 5e-4 and both diagonal multiplicative terms +|m11|, |m22| < 1%, from the full 2x2 response matrix. The grid spans galaxies +with half-light radius 0.3" and 0.5" through a 0.7" PSF, round and with +ellipticity (0.05, 0.02). A noise-filled defect biases m anisotropically +(m11 and m22 differ by up to a factor of ten), so a scalar m would hide it. +The cases sit at ``EPOCH_CENTRAL_DEFECT_RADIUS`` itself, so lowering the +radius below the calibrated value turns this red. + +Positive control: the same column two pixels inside the radius breaks the +bound, so the recovery check can see the bias the veto removes. +""" + +import json + +import numpy as np +import pytest + +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, + defect_mask, + has_central_defect, +) +from tests.helpers.defect_response import defect_response + +N = 51 +CENTRE = N // 2 +R = int(np.ceil(EPOCH_CENTRAL_DEFECT_RADIUS)) +ELLIPTICAL = (0.05, 0.02) +SEEDS = range(6) + + +def geometry(kind, distance): + """A detector defect whose nearest pixel is ``distance`` px from the + stamp centre; an edge band reaches in to that distance.""" + bad = np.zeros((N, N), dtype=bool) + if kind == "pixel": + bad[CENTRE, CENTRE + distance] = True + elif kind == "column": + bad[:, CENTRE + distance] = True + elif kind == "bleed": + bad[:, CENTRE + distance:CENTRE + distance + 3] = True + elif kind == "edge": + bad[:, CENTRE + distance:] = True + return bad + + +# Wide defects (a 3-px bleed, the widest edge band the veto keeps) at the +# radius itself on the 0.5" galaxy through the elliptical PSF sit at the +# bound (m11 = -0.98% +/- 0.04% and -0.94% +/- 0.06%; see +# :func:`has_central_defect`), so those two are checked one pixel out. +CASES = ( + [(k, d, h, (0.0, 0.0)) for k in ("column", "bleed", "pixel") + for d in (R, R + 2) for h in (0.3, 0.5)] + + [(k, R, 0.5, ELLIPTICAL) for k in ("column", "pixel")] + + [(k, R + 1, 0.5, ELLIPTICAL) for k in ("bleed", "edge")] + + [("edge", R, 0.5, (0.0, 0.0))] + + [("edge", N - CENTRE - 5, 0.5, psf) for psf in ((0.0, 0.0), ELLIPTICAL)] +) + + +def recover(bad, hlr, psf_shear, tmp_path): + result = defect_response(bad, hlr=hlr, psf=0.7, seeds=SEEDS, + psf_shear=psf_shear) + (tmp_path / "recovery.json").write_text(json.dumps(result, indent=2)) + return np.abs(result["m"]).max(), np.abs(result["c"]).max(), result + + +def case_id(case): + kind, distance, hlr, psf_shear = case + psf = "elliptical" if any(psf_shear) else "round" + return f"{kind}-{distance}px-hlr{hlr}-{psf}" + + +@pytest.mark.parametrize("kind,distance,hlr,psf_shear", CASES, + ids=[case_id(c) for c in CASES]) +def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, + psf_shear, tmp_path): + """Failure modes: the veto radius is below the calibrated value; the + fill leaks raw defect values or leaves more of the object's light + missing; the check reads m11 alone and misses m22, or c1 alone. + """ + bad = geometry(kind, distance) + masked = defect_mask(np.ones((N, N)), bad.astype(np.int32)) + np.testing.assert_array_equal(masked, bad) + assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT + assert not has_central_defect(masked, EPOCH_CENTRAL_DEFECT_RADIUS) + m, c, result = recover(bad, hlr, psf_shear, tmp_path) + assert m < 0.01, result + assert c < 5e-4, result + + +def test_vetoed_column_breaks_the_bound(tmp_path): + """Positive control: a column two pixels inside the radius, on the + 0.5" galaxy, gives |m| > 2% and |c| > 1e-3. The veto drops it.""" + bad = geometry("column", R - 2) + assert has_central_defect(bad, EPOCH_CENTRAL_DEFECT_RADIUS) + m, c, result = recover(bad, 0.5, (0.0, 0.0), tmp_path) + assert m > 0.02, result + assert c > 1e-3, result From cd8b321b2bc81e5920e9a2a9b7d501c846fff9a0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:07:40 +0200 Subject: [PATCH 02/44] feat(ngmix): noise-fill defects under every BLEND_HANDLING, veto central ones Metacal shears the whole stamp without reading the weights, so a defect left raw under uberseg reached the fit. Every defect (nonzero flag, zero weight, invalid RMS) is now zero-weighted and noise-filled whatever BLEND_HANDLING is; uberseg only zeroes the weight of neighbour-side pixels and keeps their light. The option that meant "no neighbour treatment" is renamed from noisefill to none, and the retired name is rejected. The defect set is not symmetrized: at the veto radius the one-sided fill leaves |c| <= 3e-4, while a four-fold mask would quadruple m (-2.7% against -0.64% for a 3-px bleed at 10 px on a 0.5" galaxy) and, through an elliptical PSF, still leave c1 = 7.3e-4. An epoch is dropped when a defect lies closer than EPOCH_CENTRAL_DEFECT_RADIUS (default 10 px) to the stamp centre, the smallest radius at which columns, 3-px bleeds, single pixels and edge bands give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 on 0.3" and 0.5" galaxies through a 0.7" PSF. The masked-fraction cut counts the same raw defect set; EPOCH_MASKED_FRACTION_CUT (default 1/3) makes it configurable. Each tile logs "epoch cuts: considered=N masked_fraction=A central_veto=B objects_emptied=C". The HSM centroid is measured on the filled image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- src/shapepipe/modules/ngmix_package/ngmix.py | 353 +++++++++++++++---- src/shapepipe/modules/ngmix_runner.py | 38 +- 2 files changed, 317 insertions(+), 74 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index b2e092240..807b99ff0 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -8,6 +8,7 @@ import os import re +from collections import Counter from typing import NamedTuple import ngmix @@ -25,7 +26,17 @@ from shapepipe.pipeline import file_io # Neighbour treatments selectable with the BLEND_HANDLING option. -BLEND_HANDLINGS = ("noisefill", "uberseg") +BLEND_HANDLINGS = ("none", "uberseg") + +# Default of the EPOCH_MASKED_FRACTION_CUT option: an epoch is dropped when +# more than this fraction of its stamp is in :func:`defect_mask` (see +# :func:`prepare_postage_stamps`). +EPOCH_MASKED_FRACTION_CUT = 1 / 3 + +# Default of the EPOCH_CENTRAL_DEFECT_RADIUS option (pixels): an epoch is +# dropped when a pixel of :func:`defect_mask` lies closer than this to the +# stamp centre (see :func:`has_central_defect`). +EPOCH_CENTRAL_DEFECT_RADIUS = 10 METACAL_TYPES = ('noshear', '1p', '1m', '2p', '2m') @@ -191,7 +202,7 @@ class Tile_cat(): vignetmaker output cut from the tile ``SEGMENTATION`` check image), row-aligned to ``cat_path``. When given, ``self.seg`` holds one integer seg stamp per object for the ``"uberseg"`` blend handling; ``None`` - leaves ``self.seg`` unset (the noise-fill path is unaffected). + leaves ``self.seg`` unset, which only ``"uberseg"`` needs. """ def __init__( @@ -226,7 +237,7 @@ def get_data(self, cat_path): # Coadd-frame SExtractor segmentation stamp (integer labels), one per # object and row-aligned to the tile catalogue, overlaid unchanged on # every epoch for uberseg neighbour masking (shapepipe#776). None -> - # uberseg unavailable; the noise-fill path is unaffected. + # uberseg unavailable; ``"none"`` does not read it. self.seg = None if self.seg_cat_path: seg_cat = file_io.FITSCatalogue( @@ -281,7 +292,7 @@ def __init__( self.flags = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend - # handling; empty for the default noise-fill path. All epochs carry the + # handling; empty under the default "none". All epochs carry the # SAME coadd-frame seg stamp (shapepipe#776: one coadd seg per object, # no per-epoch reprojection), each MegaCam-flipped to match its galaxy # stamp so the overlay stays registered. @@ -298,6 +309,9 @@ def __init__( # CCD number of the first epoch, used only to build the per-object # position seed (see :func:`position_seed`). self.ccd = None + self.epoch_cuts = Counter( + considered=0, masked_fraction=0, central_veto=0 + ) self.bkg_sub = bkg_sub self.megacam_flip = megacam_flip @@ -426,16 +440,26 @@ class Ngmix(object): propagated on the vignette. ``"hsm"`` re-centers on the adaptive-moment centroid measured from the stamp pixels. See :func:`make_ngmix_observation`. - blend_handling : {"noisefill", "uberseg"}, optional - Neighbour treatment; ``"noisefill"`` (default) is the historical - noise-fill, ``"uberseg"`` hard-masks neighbour-side pixels from the - coadd segmentation map and requires ``seg_cat_path``. + blend_handling : {"none", "uberseg"}, optional + Neighbour treatment. ``"none"`` (default) leaves neighbour pixels + untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels + from the coadd segmentation map and requires ``seg_cat_path``. Defect + pixels are noise-filled under both (see + :func:`prepare_ngmix_weights`). seg_cat_path : str, optional Path to the coadd-frame segmentation VIGNET catalogue (see :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. + epoch_central_defect_radius : float, optional + Drop an epoch when a masked pixel lies closer than this many pixels + to the stamp centre (see :func:`has_central_defect`); the default is + ``EPOCH_CENTRAL_DEFECT_RADIUS``. + epoch_masked_fraction_cut : float, optional + Drop an epoch when more than this fraction of its stamp is masked + (see :func:`prepare_postage_stamps`); the default is + ``EPOCH_MASKED_FRACTION_CUT``. Notes ----- @@ -466,10 +490,12 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="wcs", - blend_handling="noisefill", + blend_handling="none", seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", + epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, + epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -539,6 +565,8 @@ def __init__( self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour self._metacal_psf = metacal_psf + self._epoch_central_defect_radius = epoch_central_defect_radius + self._epoch_masked_fraction_cut = epoch_masked_fraction_cut self._w_log = w_log @@ -985,6 +1013,8 @@ def process(self): id_first = -1 id_last = -1 count_batch = 0 + epoch_cuts = Counter(considered=0, masked_fraction=0, central_veto=0) + n_emptied = 0 saved_batch_cumul = 0 for i_tile, obj_id in enumerate(tile_cat.obj_id): @@ -1014,10 +1044,14 @@ def process(self): self._bkg_sub, psf_obj, gal_obj, + epoch_central_defect_radius=self._epoch_central_defect_radius, + epoch_masked_fraction_cut=self._epoch_masked_fraction_cut, ) + epoch_cuts.update(stamp.epoch_cuts) if len(stamp.gals) == 0: n_no_epoch += 1 + n_emptied += stamp.epoch_cuts["considered"] > 0 continue # Per-object RNG, seeded from (ra, dec, ccd) — see @@ -1131,6 +1165,13 @@ def process(self): + f" {n_fitted} fitted" ) + self._w_log.info( + "epoch cuts:" + + f" considered={epoch_cuts['considered']}" + + f" masked_fraction={epoch_cuts['masked_fraction']}" + + f" central_veto={epoch_cuts['central_veto']}" + + f" objects_emptied={n_emptied}" + ) vignet_cat.close() # Put all results together @@ -1150,7 +1191,46 @@ def prepare_postage_stamps( bkg_sub=True, psf_obj=None, gal_obj=None, + epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, + epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, ): + """Gather one object's epoch stamps, dropping epochs its defects spoil. + + @sc [decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.defect_fill] epoch-cut-on-defect-mask + An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its + stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS + pixels, the set that :func:`prepare_ngmix_weights` zero-weights and + noise-fills. Counting flags alone would keep epochs whose filled area + exceeds the cut. The default is 1/3; 10%, the DES Y3 and Y6 value, is + the alternative to test. + + Parameters + ---------- + vignet : Vignet + Per-object vignet stores. + obj_id : int + Object ID (SExtractor ``NUMBER``). + i_tile : int + Row of the object in ``tile_cat``. + tile_cat : Tile_cat + Tile catalogue. + bkg_sub : bool, optional + Subtract the background vignet; the default is ``True``. + psf_obj, gal_obj : dict, optional + The object's PSF and galaxy vignet dicts, if already read. + epoch_central_defect_radius : float, optional + Drop an epoch with a defect pixel closer than this many pixels to the + stamp centre (:func:`has_central_defect`); 0 disables the veto. The + default is ``EPOCH_CENTRAL_DEFECT_RADIUS``. + epoch_masked_fraction_cut : float, optional + Drop an epoch with more than this fraction of its stamp in + :func:`defect_mask`. The default is ``EPOCH_MASKED_FRACTION_CUT``. + + Returns + ------- + Postage_stamp + The surviving epochs' stamps. + """ # define per-object lists of individual exposures to go into ngmix stamp = Postage_stamp(bkg_sub=bkg_sub) # Read each store's per-object dict ONCE: every sqlitedict access @@ -1220,17 +1300,23 @@ def prepare_postage_stamps( flag_vign = flag_obj[expccd_name]['VIGNET'] if tile_vign is not None: flag_vign[np.where(tile_vign == -1e30)] = 2**10 - v_flag_tmp = flag_vign.ravel() - # remove objects that are more than 1/3 masked - if len(np.where(v_flag_tmp != 0)[0]) / v_flag_tmp.size > 1 / 3.0: - continue - weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] if bkg_rms_obj is not None else None ) + # Drop the epoch when too much of it would be zero-weighted and + # noise-filled (epoch-cut-on-defect-mask), or when a filled pixel + # would sit near the object (epoch-central-defect-veto). + masked = defect_mask(weight_vign, flag_vign, bkg_rms_vign) + stamp.epoch_cuts["considered"] += 1 + if masked.mean() > epoch_masked_fraction_cut: + stamp.epoch_cuts["masked_fraction"] += 1 + continue + if has_central_defect(masked, epoch_central_defect_radius): + stamp.epoch_cuts["central_veto"] += 1 + continue # One unpickle per exposure (all CCDs), reused across this object's # epochs; the cache is per-call, so bounded by the object's exposures @@ -1503,9 +1589,9 @@ def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): Because the partition is by distance to the nearest footprint, the pixels surviving around a compact central object form a single connected, roughly circular core; the "circularisation" is emergent geometry, not a - separate aperture. Unlike the noise-fill treatment, masked pixels are - handed to ngmix as a hard mask (weight = 0), never replaced by a noise - realisation. + separate aperture. Only the weight changes: neighbour-side pixels are + handed to ngmix as a hard mask (weight = 0) and keep their image values, + so metacal shears the neighbour's light along with the target's. Parameters ---------- @@ -1566,33 +1652,163 @@ def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): return weight + +def defect_mask(weight, flag, bkg_rms=None): + """Defect pixels of one epoch stamp. + + @sc [decision:shape_measurement.defect_fill,label:physics] defect-set-unsymmetrized + A defect is a pixel with zero exposure weight, a nonzero flag, or (when a + background RMS map is given) a non-finite or non-positive RMS. This one + set is zero-weighted and noise-filled by :func:`prepare_ngmix_weights` + and counted by the epoch cuts in :func:`prepare_postage_stamps`. It is + not ORed with its rotations. For defects the central-defect veto keeps + (:func:`has_central_defect`), the unsymmetrized fill leaves + |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" + and 0.5" through a 0.7" PSF, round or elliptical. Symmetrizing would + quadruple the filled area near the object, and with it m: a 3-px bleed at + 10 px on the 0.5" galaxy gives m11 = -2.7% four-fold against -0.64% + unsymmetrized. Through a PSF with ellipticity (0.05, 0.02) it would not + cancel c either: c1 = 7.3e-4 four-fold against 0.7e-4. It would also + turn a 5-px edge band (9.8% of the stamp), which biases nothing, into a + 35.4% frame that fails the masked-fraction cut. + + Parameters + ---------- + weight : numpy.ndarray + Exposure weight stamp. + flag : numpy.ndarray + Flag stamp. + bkg_rms : numpy.ndarray, optional + Background RMS stamp. + + Returns + ------- + numpy.ndarray of bool + ``True`` on defect pixels. + """ + defect = (weight == 0) | (flag != 0) + if bkg_rms is not None: + defect |= ~(np.isfinite(bkg_rms) & (bkg_rms > 0)) + return defect + + +def has_central_defect(defect, radius): + """Whether a defect pixel lies closer than ``radius`` to the stamp centre. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.defect_fill] epoch-central-defect-veto + An epoch is dropped when any pixel of :func:`defect_mask` lies closer + than ``radius`` pixels to the stamp centre. A noise-filled hole in the + object's light is sheared by metacal but not by the sky, so the metacal + response is wrong for that epoch, and a one-sided hole adds an additive + term. The default radius (``EPOCH_CENTRAL_DEFECT_RADIUS``, 10 px or + 1.9") is the smallest at which columns, 3-px bleeds, single pixels and + edge bands give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 (full response + matrix) on galaxies with half-light radius 0.3" and 0.5" through a 0.7" + PSF. The bias is anisotropic: a column at 10 px on the 0.5" galaxy gives + m11 = -0.60%, m22 = +0.01% and c1 = -1.4e-4; at 9 px, m11 = -1.75% and + c1 = -1.1e-3. Through a PSF with ellipticity (0.05, 0.02), wide defects + at 10 px on the 0.5" galaxy sit at the bound: a 3-px bleed gives + m11 = -0.98% +/- 0.04% and the widest edge band the veto keeps (16 px) + -0.94% +/- 0.06%; at 11 px the bleed gives -0.24%. Larger galaxies need + more: at hlr 0.7" through a 0.9" PSF a 3-px bleed at 10 px gives + m11 = -6.8% and c1 = -4.9e-3, and passes from 14 px. The distance is + measured from the stamp centre, where the extractor places the object to + within half a pixel. The veto reads only the defect mask, never the + object's pixels, so it selects on nothing that responds to shear. + Radius 0 disables it. Guarded by ``tests/science/test_defect_veto.py``. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp (:func:`defect_mask`). + radius : float + Veto radius in pixels. + + Returns + ------- + bool + ``True`` if the epoch should be dropped. + """ + rows, cols = np.nonzero(defect) + centre_row = (defect.shape[0] - 1) / 2 + centre_col = (defect.shape[1] - 1) / 2 + distance = np.hypot(rows - centre_row, cols - centre_col) + return bool(np.any(distance < radius)) + + +def fill_defects(image, defect, noise): + """Replace an image's defect pixels by a noise realisation. + + @sc [decision:shape_measurement.defect_fill,label:physics] defect-noise-fill + Defect pixels are replaced by ``noise``, an independent realisation at + the per-pixel background RMS. Metacal deconvolves, shears and reconvolves + the whole image without reading the weights, so a raw defect value (bad + column, bleed, cosmic ray) would leak into the fit. The noise fill leaves + a hole in the object's light that metacal shears and the sky does not, + which biases m for an epoch with a defect near the object; the + central-defect veto drops those epochs (epoch-central-defect-veto). + + Parameters + ---------- + image : numpy.ndarray + Stamp image. + defect : numpy.ndarray of bool + Pixels to replace (:func:`defect_mask`). + noise : numpy.ndarray + Noise realisation on the stamp grid. + + Returns + ------- + numpy.ndarray + ``image`` with ``defect`` pixels taken from ``noise``. + """ + return np.where(defect, noise, image) + + def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, - blend_handling="noisefill", seg=None, object_number=None, + blend_handling="none", seg=None, object_number=None, dilate_neighbour=0, ): - """bookkeeping for ngmix weights. runs on a single galaxy and epoch - pixel scale and galaxy guess - TO DO: decide if we want galaxy guess stuff + """Build one epoch's image, weight map and noise image for ngmix. + + Defect pixels (:func:`defect_mask`: flagged, zero-weight or invalid-RMS + pixels) get weight 0 and are replaced by an independent noise + realisation at their background RMS (:func:`fill_defects`), under either + ``blend_handling``. ``blend_handling`` decides only the neighbour side. + + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-neighbours-raw + Every pixel of ``defect_mask`` is zero-weighted and noise-filled whatever + ``blend_handling`` is, and the filled set equals the zero-weight defect + set. ``blend_handling`` acts on the neighbour side only: uberseg zeroes + the weight of pixels nearer a neighbour's footprint and leaves their + image values raw, because that light is real sky that metacal shears + along with the target; filling it with noise would cut the target along + an unsheared edge. Neighbour pixels are never noise-filled. The noise + image covers the whole stamp and is independent of both. Parameters ---------- gal : numpy.ndarray + Background-subtracted galaxy stamp. weight : numpy.ndarray + Exposure weight stamp; zero marks a defect. flag : numpy.ndarray + Flag stamp; nonzero marks a defect. rng : numpy.random.RandomState Random state for the noise realisations (seeded per object; see :func:`position_seed`). bkg_rms : numpy.ndarray, optional - Per-pixel background RMS map. If supplied, unmasked pixels use - ``1 / bkg_rms**2`` as the ngmix inverse variance. - blend_handling : {"noisefill", "uberseg"}, optional - How to treat pixels shared with a neighbour. ``"noisefill"`` (default) - replaces flagged pixels with a noise realisation and keeps their - inverse-variance weight — the historical behaviour. ``"uberseg"`` - instead hard-masks (weight = 0) every pixel closer to a neighbour's - segmentation footprint than to the central object's, leaving the - image untouched (see :func:`uberseg_weight`). + Per-pixel background RMS map. If supplied, clean pixels use + ``1 / bkg_rms**2`` as the ngmix inverse variance, and non-finite or + non-positive values mark defects. Otherwise every clean pixel gets + ``1 / sigma_mad(gal)**2``. + blend_handling : {"none", "uberseg"}, optional + Neighbour treatment. ``"none"`` (default) leaves neighbour + pixels weighted and untouched. ``"uberseg"`` zeroes the weight of + every pixel closer to a neighbour's segmentation footprint than to the + central object's and keeps its raw image value (see + :func:`uberseg_weight`). seg : numpy.ndarray, optional Segmentation map on the stamp grid (object NUMBERs). Required for ``blend_handling="uberseg"``; ignored otherwise. @@ -1606,44 +1822,57 @@ def prepare_ngmix_weights( Returns ------- numpy.ndarray - Galaxy image. For ``"noisefill"`` masked pixels are replaced by noise; - for ``"uberseg"`` the image is returned untouched. + Galaxy image with defect pixels noise-filled. numpy.ndarray - Variance map for NGMIX. + Inverse-variance weight map for ngmix. numpy.ndarray - Noise image. + Noise image: an independent realisation over the whole stamp, for + metacal's ``fixnoise``. + + Raises + ------ + ValueError + If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or + ``object_number``. """ if blend_handling not in BLEND_HANDLINGS: raise ValueError( f"Unknown blend_handling '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) + if blend_handling == "uberseg" and (seg is None or object_number is None): + raise ValueError( + "blend_handling='uberseg' requires a segmentation map and the" + + " central object_number; none reached prepare_ngmix_weights." + + " Set SEG_VIGNET_PATH on the ngmix run (see" + + " CosmoStat/shapepipe#776)." + ) - mask = np.copy(weight) != 0 - mask[flag != 0] = False + defect = defect_mask(weight, flag, bkg_rms) + clean = ~defect if bkg_rms is None: sig_noise = sigma_mad(gal) # Guard the degenerate constant stamp (sigma_mad == 0): 0 * inf # would otherwise put NaN in a fully-masked weight map. weight_map = ( - mask.astype(float) / sig_noise ** 2 + clean.astype(float) / sig_noise ** 2 if sig_noise > 0 else np.zeros_like(gal, dtype=float) ) else: - valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) - mask &= valid_rms weight_map = np.zeros_like(gal, dtype=float) - weight_map[mask] = 1.0 / bkg_rms[mask] ** 2 + weight_map[clean] = 1.0 / bkg_rms[clean] ** 2 # Per-pixel noise sigma for the realisations below: metacal's # fixnoise bookkeeping (1/w + 1/w_noise) assumes the noise image # is a faithful realisation of the per-pixel variance the weights # claim; a scalar sigma there mis-reports errors and erodes the # inverse-variance advantage whenever the RMS map actually varies. + # Pixels without a valid RMS take the median over clean pixels. + valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) sig_noise = ( - np.where(valid_rms, bkg_rms, np.median(bkg_rms[mask])) - if mask.any() + np.where(valid_rms, bkg_rms, np.median(bkg_rms[clean])) + if clean.any() else sigma_mad(gal) ) @@ -1656,33 +1885,20 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise + gal_filled = fill_defects(gal, defect, noise_img_gal) - gal_masked = np.copy(gal) if blend_handling == "uberseg": - # Hard-mask neighbour-side pixels (weight -> 0) from the segmentation - # geometry; the image is left untouched (the masked pixels carry no - # weight, so ngmix ignores them in the likelihood). Bad/flagged - # pixels already sit at weight 0 from the mask above. - if seg is None or object_number is None: - raise ValueError( - "blend_handling='uberseg' requires a segmentation map and the" - + " central object_number; none reached prepare_ngmix_weights." - + " Set SEG_VIGNET_PATH on the ngmix run (see" - + " CosmoStat/shapepipe#776)." - ) weight_map = uberseg_weight( weight_map, seg, object_number, dilate_neighbour=dilate_neighbour ) - elif (~mask).any(): - # noisefill (default): replace masked pixels with a noise realisation. - gal_masked[~mask] = noise_img_gal[~mask] - return gal_masked, weight_map, noise_img + return gal_filled, weight_map, noise_img + def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, - blend_handling="noisefill", seg=None, object_number=None, + blend_handling="none", seg=None, object_number=None, dilate_neighbour=0, ): """Build an ngmix Observation for a single galaxy epoch. @@ -1725,9 +1941,9 @@ def make_ngmix_observation( Sub-pixel ``[row, col]`` coadd-centroid offset propagated from the stamp extractor. Required for ``centroid_source="wcs"`` (ignored for ``"hsm"``). - blend_handling : {"noisefill", "uberseg"}, optional + blend_handling : {"none", "uberseg"}, optional Neighbour treatment passed through to :func:`prepare_ngmix_weights`; - the default ``"noisefill"`` is the historical behaviour. + the default ``"none"`` leaves neighbour pixels untouched. seg : numpy.ndarray, optional Segmentation map on the stamp grid. Required for ``blend_handling="uberseg"`` (ignored otherwise). @@ -1769,11 +1985,12 @@ def make_ngmix_observation( if centroid_source == "hsm": # Re-center the Jacobian on the HSM adaptive-moment centroid (pixel - # offset from the stamp center); fall back to the stamp center if - # HSM fails. + # offset from the stamp center), measured on the filled image so no + # raw defect value pulls it; fall back to the stamp center if HSM + # fails. try: _hsm = galsim.hsm.FindAdaptiveMom( - galsim.Image(gal, scale=1.0), strict=False + galsim.Image(gal_masked, scale=1.0), strict=False ) if _hsm.error_message != "": raise galsim.hsm.GalSimHSMError(_hsm.error_message) @@ -1988,7 +2205,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", - blend_handling="noisefill", object_number=None, dilate_neighbour=0, + blend_handling="none", object_number=None, dilate_neighbour=0, metacal_psf="fitgauss", ): """Do Ngmix Metacal. @@ -2012,10 +2229,10 @@ def do_ngmix_metacal( coadd-centroid offset in ``stamp.offsets``, propagated from the stamp extractor); ``"hsm"`` uses the adaptive-moment centroid from the stamp pixels — see that function. - blend_handling : {"noisefill", "uberseg"}, optional + blend_handling : {"none", "uberseg"}, optional Neighbour treatment passed through to - :func:`make_ngmix_observation`; the default ``"noisefill"`` is the - historical behaviour. ``"uberseg"`` consumes ``stamp.segs`` and + :func:`make_ngmix_observation`; the default ``"none"`` leaves + neighbour pixels untouched. ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional Central object's segmentation label — its SExtractor ``NUMBER`` diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 27648947f..ffc4de083 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -11,7 +11,11 @@ from sqlitedict import SqliteDict from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.ngmix_package.ngmix import Ngmix +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, + Ngmix, +) @module_runner( @@ -125,14 +129,16 @@ def ngmix_runner( else: centroid_source = "wcs" - # Neighbour treatment: "noisefill" (default, historical) replaces a - # neighbour's pixels with a noise realisation; "uberseg" hard-masks - # (weight -> 0) every pixel closer to a neighbour than to the central - # object, from the segmentation map. See the ngmix module docstrings. + # Neighbour treatment: "none" (default) leaves neighbour pixels + # weighted and untouched; "uberseg" zeroes the weight of every pixel + # closer to a neighbour than to the central object, from the segmentation + # map, and leaves its image raw. Defect pixels (flagged, zero-weight or + # invalid-RMS) are zero-weighted and noise-filled under both; see + # prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: - blend_handling = "noisefill" + blend_handling = "none" # DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the # uberseg neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay @@ -142,6 +148,24 @@ def ngmix_runner( else: dilate_neighbour = 1 + # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a + # masked pixel lies closer than this to the stamp centre; 0 disables. + if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): + epoch_central_defect_radius = config.getfloat( + module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" + ) + else: + epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS + + # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this + # fraction of its stamp is masked (flagged, zero-weight or invalid-RMS). + if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): + epoch_masked_fraction_cut = config.getfloat( + module_config_sec, "EPOCH_MASKED_FRACTION_CUT" + ) + else: + epoch_masked_fraction_cut = EPOCH_MASKED_FRACTION_CUT + # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -197,6 +221,8 @@ def ngmix_runner( seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, metacal_psf=metacal_psf, + epoch_central_defect_radius=epoch_central_defect_radius, + epoch_masked_fraction_cut=epoch_masked_fraction_cut, ) # Process ngmix shape measurement and metacalibration From 7777181bc6cd76f3c0b7a41d4b98bccb218442b1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:07:40 +0200 Subject: [PATCH 03/44] ngmix: read each epoch's OFFSET from the object dict in hand prepare_postage_stamps already holds the object's galaxy vignet dict, so the per-epoch OFFSET is read from it instead of a second sqlitedict lookup. The pixel scale sets only the centroid-prior width; the comments that also credited it with a noise window are corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- .../modules/ngmix_package/__init__.py | 4 ++-- src/shapepipe/modules/ngmix_package/ngmix.py | 18 ++++++++---------- src/shapepipe/modules/ngmix_runner.py | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 66323822c..012b3264b 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -36,8 +36,8 @@ PIXEL_SCALE : float, optional Pixel scale in arcsec. Optional override; when omitted (or non-positive) it is read from the image WCS so it cannot drift from the pixels. Only - sets the centroid-prior width and noise window -- the fit Jacobian is - built per object from the full WCS. + sets the centroid-prior width -- the fit Jacobian is built per object + from the full WCS. SAVE_BATCH : int, optional Save the output catalogue in batches of this size; default is ``-1`` (no batch saving) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 807b99ff0..a19d2a83e 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -367,11 +367,11 @@ def pixel_scale_from_wcs(f_wcs_file): """Representative pixel scale (arcsec) from the tile's image WCS. The ngmix fit builds each object's Jacobian from the full per-epoch WCS, - so this scalar only sets the centroid-prior width and the noise-window - scale (see :func:`get_prior`, :func:`get_noise`). A single value read from - the astrometry is therefore sufficient -- and, unlike a hard-coded config - constant, it cannot silently drift from the pixels it describes (this - mirrors SExtractor's ``PIXEL_SCALE 0`` convention). + so this scalar only sets the centroid-prior width (see :func:`get_prior`). + A single value read from the astrometry is therefore sufficient -- and, + unlike a hard-coded config constant, it cannot silently drift from the + pixels it describes (this mirrors SExtractor's ``PIXEL_SCALE 0`` + convention). Parameters ---------- @@ -578,8 +578,8 @@ def __init__( # Pixel scale: an explicit positive PIXEL_SCALE overrides; otherwise # derive it from the image WCS so it can never drift from the pixels # (mirrors SExtractor's ``PIXEL_SCALE 0`` convention). Only the - # centroid-prior width and noise window use it -- the fit Jacobian is - # built per object from the full WCS. + # centroid-prior width uses it -- the fit Jacobian is built per + # object from the full WCS. if pixel_scale is None or pixel_scale <= 0: self._pixel_scale = pixel_scale_from_wcs( self._vignet_cat.f_wcs_file @@ -1359,9 +1359,7 @@ def prepare_postage_stamps( # make_ngmix_observation), which raises if it is missing; the "hsm" # path ignores it, so read it leniently rather than coupling hsm to a # field it never uses. - stamp.offsets.append( - vignet.gal_vign_cat[str(obj_id)][expccd_name].get('OFFSET') - ) + stamp.offsets.append(gal_obj[expccd_name].get('OFFSET')) stamp.ra.append(tile_cat.ra[i_tile]) stamp.dec.append(tile_cat.dec[i_tile]) # CCD of the first surviving epoch — Fabian's coord_list[0] convention diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index ffc4de083..605262395 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -54,7 +54,7 @@ def ngmix_runner( # Pixel scale -- optional override. When absent (or non-positive) it is # derived from the image WCS inside Ngmix, so it cannot drift from the - # pixels. Only the centroid-prior width and noise window use it. + # pixels. Only the centroid-prior width uses it. if config.has_option(module_config_sec, "PIXEL_SCALE"): pixel_scale = config.getfloat(module_config_sec, "PIXEL_SCALE") else: From 9f0ea4430f32f998a3a9e83fd1466cbeab69546d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:44:32 +0200 Subject: [PATCH 04/44] tests: name the none-mode uberseg test for none, not noisefill Co-Authored-By: Claude Opus 5.5 --- tests/module/test_ngmix_uberseg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 86dad33bc..4d6ebb32a 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -239,9 +239,9 @@ def _gal_flag_weight(npix=41, seed=7): return gal, flag, weight -def test_noisefill_ignores_seg_and_dilate_kwargs(): - """Under noisefill, passing seg / dilate_neighbour changes nothing: the - result matches the plain default call on the same RNG stream.""" +def test_none_ignores_seg_and_dilate_kwargs(): + """Under BLEND_HANDLING = none, passing seg / dilate_neighbour changes + nothing: the result matches the plain default call on the same RNG stream.""" gal, flag, weight = _gal_flag_weight() seg, _, _ = two_object_seg(npix=gal.shape[0], sep=12) From a92a7471fbb09a067512ffb8f13a7c775426cac1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:22:49 +0200 Subject: [PATCH 05/44] test(ngmix): specify DEFECT_FILL = interpolate and its shear recovery The interpolation fills only short defect runs (at most 3 px along a row or column, with clean pixels at both ends); anything wider, and runs reaching the stamp border, stays noise-filled. The interpolant must reproduce linear planes without reading defect values, commute with quarter turns of the stamp, and apply one operator to the science and metacal noise images. The weight is zero on the defects and on the quarter-turn orbit of the interpolated pixels, whose light stays. Each defect is vetoed at the radius of its own fill: EPOCH_INTERPOLATED_DEFECT_RADIUS for interpolated pixels, EPOCH_CENTRAL_DEFECT_RADIUS for noise-filled ones. The runner and Ngmix pass DEFECT_FILL to the epoch cuts and to metacal; noise stays the default. The science test recovers the full response matrix for interpolated columns, full and finite 3-px bleeds and single pixels at the interpolated radius, on 0.3" and 0.5" galaxies through round and elliptical 0.7" PSFs and on a 0.7" galaxy through a 0.9" PSF, and for noise-filled wide defects at the noise radius. A bleed three pixels inside the radius is the positive control. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- tests/module/test_defect_interpolation.py | 237 +++++++++++++++++++++ tests/module/test_ngmix_defect_fill.py | 223 ++++++++++++++++++- tests/science/test_defect_interpolation.py | 119 +++++++++++ 3 files changed, 568 insertions(+), 11 deletions(-) create mode 100644 tests/module/test_defect_interpolation.py create mode 100644 tests/science/test_defect_interpolation.py diff --git a/tests/module/test_defect_interpolation.py b/tests/module/test_defect_interpolation.py new file mode 100644 index 000000000..1368e78cc --- /dev/null +++ b/tests/module/test_defect_interpolation.py @@ -0,0 +1,237 @@ +"""Defect interpolation (``DEFECT_FILL = interpolate``): which pixels are +interpolated, and the properties of the interpolant. + +:func:`interpolable_defects` picks the defect pixels that lie in a short row +or column run with clean pixels at both ends; :func:`interpolate_defects` +fills them from the nearby clean pixels with a Clough-Tocher interpolant, +averaged over the four quarter turns of the stamp and shared by every plane +(science image and metacal noise image). +""" + +import numpy as np +import numpy.testing as npt +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from shapepipe.modules.ngmix_package.defect_interpolation import ( + MAX_INTERPOLATED_RUN, + fourfold, + interpolable_defects, + interpolate_defects, +) + + +# --- interpolable_defects --------------------------------------------------- + +def _oracle(defect, max_run=MAX_INTERPOLATED_RUN): + """Brute force: walk each defect pixel's row and column run.""" + n0, n1 = defect.shape + out = np.zeros_like(defect) + for i, j in zip(*np.nonzero(defect)): + for di, dj in ((0, 1), (1, 0)): + lo, hi = (i, j), (i, j) + while (0 <= lo[0] - di and 0 <= lo[1] - dj + and defect[lo[0] - di, lo[1] - dj]): + lo = (lo[0] - di, lo[1] - dj) + while (hi[0] + di < n0 and hi[1] + dj < n1 + and defect[hi[0] + di, hi[1] + dj]): + hi = (hi[0] + di, hi[1] + dj) + length = hi[0] - lo[0] + hi[1] - lo[1] + 1 + bounded = (lo[0] - di >= 0 and lo[1] - dj >= 0 + and hi[0] + di < n0 and hi[1] + dj < n1) + if bounded and length <= max_run: + out[i, j] = True + return out + + +@given( + n=st.integers(5, 21), + density=st.floats(0.0, 0.6), + seed=st.integers(0, 2**31 - 1), +) +@settings(max_examples=60, deadline=None) +def test_interpolable_defects_are_the_short_bounded_runs(n, density, seed): + """A defect pixel is interpolated exactly when its row or column run is + at most MAX_INTERPOLATED_RUN long and has clean pixels at both ends; the + rule commutes with quarter turns. + + Failure modes: a run touching the stamp border (edge band, corner) or a + wide hole is interpolated from one side; a 3-px bleed is left to noise; + a clean pixel is selected; one axis is ignored, so the rule has a + preferred direction. + """ + defect = np.random.RandomState(seed).uniform(size=(n, n)) < density + out = interpolable_defects(defect) + npt.assert_array_equal(out, _oracle(defect)) + assert not out[~defect].any() + for k in range(1, 4): + npt.assert_array_equal( + interpolable_defects(np.rot90(defect, k)), np.rot90(out, k) + ) + + +@pytest.mark.parametrize("kind,expected", [ + ("column", True), ("bleed3", True), ("finite_bleed3", True), + ("pixel", True), ("bleed4", False), ("blob5", False), + ("edge_band", False), ("corner", False), +]) +def test_calibrated_widths_are_interpolated_and_wider_holes_are_not( + kind, expected, +): + """Columns, 3-px bleeds and single pixels are interpolated; 4-px bleeds, + blobs, edge bands and corners are not.""" + n, c = 51, 25 + defect = np.zeros((n, n), dtype=bool) + region = { + "column": np.s_[:, c + 8], + "bleed3": np.s_[:, c + 8:c + 11], + "finite_bleed3": np.s_[c - 5:c + 6, c + 8:c + 11], + "pixel": np.s_[c, c + 8], + "bleed4": np.s_[:, c + 8:c + 12], + "blob5": np.s_[c - 2:c + 3, c + 8:c + 13], + "edge_band": np.s_[:, -3:], + "corner": np.s_[:2, :2], + }[kind] + defect[region] = True + out = interpolable_defects(defect) + assert out[defect].all() if expected else not out.any() + + +# --- fourfold --------------------------------------------------------------- + +@given(n=st.integers(2, 20), seed=st.integers(0, 2**31 - 1)) +@settings(max_examples=30, deadline=None) +def test_fourfold_is_the_quarter_turn_orbit(n, seed): + """The union contains the mask, is invariant under quarter turns, and is + the smallest such set (idempotent).""" + mask = np.random.RandomState(seed).uniform(size=(n, n)) < 0.1 + out = fourfold(mask) + assert out[mask].all() + for k in range(1, 4): + npt.assert_array_equal(np.rot90(out, k), out) + npt.assert_array_equal(fourfold(out), out) + npt.assert_array_equal( + out, mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) + ) + + +def test_fourfold_rejects_rectangular_stamps(): + with pytest.raises(ValueError, match="square"): + fourfold(np.zeros((11, 12), dtype=bool)) + + +# --- interpolate_defects ---------------------------------------------------- + +def _mask(n=31): + """A column, a finite 3-px bleed and a single pixel, all bounded.""" + defect = np.zeros((n, n), dtype=bool) + defect[:, 19] = True + defect[4:12, 7:10] = True + defect[22, 11] = True + return defect + + +@given(seed=st.integers(0, 2**31 - 1)) +@settings(max_examples=20, deadline=None) +def test_interpolation_reproduces_planes_without_reading_defects(seed): + """Linear planes are reproduced at the interpolated pixels; clean pixels, + and defect pixels not selected, are returned untouched; the inputs are + not modified; and no defect value (NaN or a sentinel) is ever read. + + Failure modes: defect pixels enter the support; coordinates are + transposed or mis-rotated; the fill smooths clean pixels. + """ + n = 31 + rng = np.random.RandomState(seed) + rows, cols = np.indices((n, n)) + coeff = rng.uniform(-2, 2, (2, 3)) + planes = np.array([a + b * rows + c * cols for a, b, c in coeff]) + defect = _mask(n) + defect[:, -2:] = True # an edge band and a blob next to the column: + defect[13:18, 21:26] = True # defects that are not targets + target = interpolable_defects(defect) + assert target[15, 19] + assert not target[13:18, 21:26].any() and not target[:, -2:].any() + contaminated = planes.copy() + contaminated[:, defect] = np.nan + saved = contaminated.copy() + + out = interpolate_defects(contaminated, defect, target) + + npt.assert_allclose(out[:, target], planes[:, target], atol=1e-5) + npt.assert_array_equal(out[:, ~target], contaminated[:, ~target]) + npt.assert_array_equal(contaminated, saved) + contaminated[:, defect] = 1e30 + npt.assert_array_equal( + interpolate_defects(contaminated, defect, target)[:, target], + out[:, target], + ) + + +def test_interpolation_commutes_with_quarter_turns(): + """Rotating the stamp and its mask rotates the fill. A regular grid's + Delaunay triangulation has degenerate diagonals, so a single-orientation + interpolant does not commute; the four-orientation average does. + + Failure mode: the average over orientations is skipped, so the fill has a + preferred direction. + """ + n = 31 + rows, cols = np.indices((n, n)) + image = np.exp(-((rows - 15.3) ** 2 + (cols - 14.6) ** 2) / 10.0) + image += 0.05 * np.random.RandomState(3).normal(size=(n, n)) + planes = image[None] + defect = _mask(n) + target = interpolable_defects(defect) + out = interpolate_defects(planes, defect, target) + for k in range(1, 4): + rotated = interpolate_defects( + np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), + np.rot90(target, k), + ) + npt.assert_allclose( + rotated, np.rot90(out, k, axes=(1, 2)), atol=1e-12, rtol=0 + ) + + +def test_every_plane_sees_the_same_operator(): + """The fill is one linear operator applied to every plane: filling + a * image + b * noise gives a * fill(image) + b * fill(noise), up to the + Clough-Tocher gradient solver's tolerance. + + Failure mode: the noise image is filled differently from the science + image (another support, triangulation or orientation set), so metacal's + fixnoise no longer mirrors the science image's correlated noise. + """ + n = 31 + rng = np.random.RandomState(11) + image, noise = rng.normal(size=(2, n, n)) + defect = _mask(n) + target = interpolable_defects(defect) + both = interpolate_defects(np.array([image, noise]), defect, target) + mixed = interpolate_defects( + np.array([2.0 * image - 3.0 * noise]), defect, target + ) + npt.assert_allclose(mixed[0], 2.0 * both[0] - 3.0 * both[1], atol=1e-5) + alone = interpolate_defects(noise[None], defect, target) + npt.assert_allclose(alone[0], both[1], atol=1e-5) + + +def test_no_target_is_a_no_op(): + planes = np.random.RandomState(5).normal(size=(2, 15, 15)) + defect = np.zeros((15, 15), dtype=bool) + defect[:, -3:] = True + out = interpolate_defects(planes, defect, np.zeros_like(defect)) + npt.assert_array_equal(out, planes) + + +def test_degenerate_support_gives_nan_not_an_error(): + """With fewer than three non-collinear clean pixels the target is NaN, + which the caller replaces by noise.""" + defect = np.ones((5, 5), dtype=bool) + defect[2, 1] = defect[2, 3] = False + target = np.zeros_like(defect) + target[2, 2] = True + out = interpolate_defects(np.ones((1, 5, 5)), defect, target) + assert np.isnan(out[0, 2, 2]) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 0480c7ef7..cbbbaa3b0 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -2,12 +2,14 @@ A defect is a stamp pixel with a nonzero flag, zero exposure weight or an invalid background RMS. Before metacal, :func:`prepare_ngmix_weights` gives -every defect weight 0 and fills it with noise at its background RMS, whatever -``BLEND_HANDLING`` is. Under uberseg, pixels on the neighbour side only lose -their weight, and their image values stay raw. The per-epoch cuts in -:func:`prepare_postage_stamps` act on the same defect set: the masked-fraction -cut counts it, and the central-defect veto drops an epoch with a defect near -the stamp centre. +every defect weight 0 and fills it, whatever ``BLEND_HANDLING`` is: with noise +at its background RMS (``DEFECT_FILL = noise``, the default) or, for short +defect runs, with an interpolant of the clean pixels around them +(``DEFECT_FILL = interpolate``). Under uberseg, pixels on the neighbour side +only lose their weight, and their image values stay raw. The per-epoch cuts +in :func:`prepare_postage_stamps` act on the same defect set: the +masked-fraction cut counts it, and the central-defect veto drops an epoch +with a defect near the stamp centre, at a radius set by the defect's fill. """ import re @@ -15,6 +17,7 @@ import numpy as np import numpy.testing as npt +import pytest from astropy.io import fits from astropy.wcs import WCS from hypothesis import given @@ -22,8 +25,14 @@ from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package import ngmix as ngmix_module +from shapepipe.modules.ngmix_package.defect_interpolation import ( + fourfold, + interpolable_defects, + interpolate_defects, +) from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, Ngmix, make_ngmix_observation, prepare_ngmix_weights, @@ -461,8 +470,9 @@ def getboolean(self, _sec, key, fallback=False): def test_runner_threads_the_epoch_cut_options(tmp_path, monkeypatch): - """EPOCH_CENTRAL_DEFECT_RADIUS and EPOCH_MASKED_FRACTION_CUT reach Ngmix - as configured, and default to the module constants when absent. + """EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_MASKED_FRACTION_CUT, DEFECT_FILL and + EPOCH_INTERPOLATED_DEFECT_RADIUS reach Ngmix as configured, and default + to the module constants (and the noise fill) when absent. Failure mode: the runner drops or ignores an option, so a configured A/B arm silently runs the default cuts. @@ -483,11 +493,15 @@ def process(self): for path in inputs: SqliteDict(path).close() - for options, radius, fraction in ( + for options, radius, fraction, fill, interpolated_radius in ( ({"EPOCH_CENTRAL_DEFECT_RADIUS": "7.5", - "EPOCH_MASKED_FRACTION_CUT": "0.1"}, 7.5, 0.1), + "EPOCH_MASKED_FRACTION_CUT": "0.1", + "DEFECT_FILL": "interpolate", + "EPOCH_INTERPOLATED_DEFECT_RADIUS": "5.5"}, + 7.5, 0.1, "interpolate", 5.5), ({}, EPOCH_CENTRAL_DEFECT_RADIUS, - ngmix_module.EPOCH_MASKED_FRACTION_CUT), + ngmix_module.EPOCH_MASKED_FRACTION_CUT, "noise", + EPOCH_INTERPOLATED_DEFECT_RADIUS), ): runner_module.ngmix_runner( inputs, {"output": str(tmp_path)}, "-001-001", @@ -495,6 +509,9 @@ def process(self): ) assert captured[-1]["epoch_central_defect_radius"] == radius assert captured[-1]["epoch_masked_fraction_cut"] == fraction + assert captured[-1]["defect_fill"] == fill + assert (captured[-1]["epoch_interpolated_defect_radius"] + == interpolated_radius) # --- make_ngmix_observation: the HSM centroid reads the filled image ------- @@ -530,3 +547,187 @@ def test_hsm_centroid_ignores_raw_defect_values(): npt.assert_allclose( [row - centre, col - centre], [d_row, d_col], atol=0.05 ) + + +# --- DEFECT_FILL = interpolate ---------------------------------------------- + +def _hot_stamp(): + """A bright object with hot defects: a column and a finite 3-px bleed + (interpolated), an edge band and a 5x5 blob (noise-filled).""" + centre = N_STAMP // 2 + rows, cols = np.mgrid[:N_STAMP, :N_STAMP] + gal = 1e3 * np.exp( + -((rows - centre) ** 2 + (cols - centre) ** 2) / (2 * 3.0 ** 2) + ) + flag = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + flag[:, centre + 8] = 1 + flag[centre - 5:centre + 6, centre - 11:centre - 8] = 1 + flag[:, :4] = 1 + flag[centre + 9:centre + 14, centre + 12:centre + 17] = 1 + gal[flag != 0] = 5e4 + return gal, np.ones((N_STAMP, N_STAMP)), flag + + +@pytest.mark.parametrize("blend_handling", ["none", "uberseg"]) +def test_interpolated_fill_and_its_weights(blend_handling): + """Short defect runs take the interpolant of the clean image; other + defects take noise; the weight is zero on the defects and on the + quarter-turn orbit of the interpolated pixels, whose light stays; the + metacal noise image is interpolated with the same operator. + + Failure modes: the orbit is not zero-weighted (a one-sided hole in the + likelihood biases c), or its light is replaced; the fill mask is + symmetrized; wide defects or edge bands are extrapolated; raw defect + values leak; the noise image keeps independent noise where the science + image is smooth. + """ + gal, weight, flag = _hot_stamp() + defect = flag != 0 + target = interpolable_defects(defect) + assert target.any() and (defect & ~target).any() + kwargs = ( + dict(seg=_uberseg_seg(N_STAMP), object_number=1, dilate_neighbour=1) + if blend_handling == "uberseg" + else {} + ) + gal_out, w_out, noise_out = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(4), + bkg_rms=np.ones((N_STAMP, N_STAMP)), blend_handling=blend_handling, + defect_fill="interpolate", **kwargs, + ) + + neighbour = ( + uberseg_weight(np.ones_like(gal), kwargs["seg"], 1, dilate_neighbour=1) + == 0.0 + if kwargs else np.zeros_like(defect) + ) + npt.assert_array_equal(w_out == 0.0, defect | fourfold(target) | neighbour) + npt.assert_array_equal(gal_out[~defect], gal[~defect]) + expected = interpolate_defects(gal[None], defect, target)[0] + npt.assert_allclose(gal_out[target], expected[target], rtol=1e-5) + assert np.all(np.abs(gal_out[defect & ~target]) < 10.0) + refilled = interpolate_defects(noise_out[None], defect, target)[0] + npt.assert_allclose(noise_out[target], refilled[target], atol=1e-5) + + +def test_noise_is_the_default_fill(): + """Leaving DEFECT_FILL unset is bit-identical to the noise fill.""" + gal, weight, flag = _hot_stamp() + rms = np.ones_like(gal) + default = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(9), bkg_rms=rms + ) + noise = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(9), bkg_rms=rms, + defect_fill="noise", + ) + for a, b in zip(default, noise): + npt.assert_array_equal(a, b) + + +def test_unknown_defect_fill_is_rejected(): + gal, weight, flag = _hot_stamp() + with pytest.raises(ValueError, match="DEFECT_FILL"): + prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + defect_fill="interp", + ) + + +def _fill_veto_epochs(): + """A clean epoch; a column at the interpolated-defect radius; a column + one pixel inside it; a 5-px bleed (noise-filled) at the same radius.""" + centre = N_STAMP // 2 + radius = int(EPOCH_INTERPOLATED_DEFECT_RADIUS) + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + at, inside, wide = clean.copy(), clean.copy(), clean.copy() + at[:, centre + radius] = 1 + inside[:, centre + radius - 1] = 1 + wide[:, centre + radius:centre + radius + 5] = 1 + return { + "2100001-10": (clean, ones), + "2100002-11": (at, ones), + "2100003-12": (inside, ones), + "2100004-13": (wide, ones), + } + + +def test_the_veto_radius_follows_the_fill(): + """Under interpolation, an interpolated defect is vetoed inside + EPOCH_INTERPOLATED_DEFECT_RADIUS and a noise-filled one inside + EPOCH_CENTRAL_DEFECT_RADIUS. Under the noise fill every defect keeps the + noise radius. The masked-fraction cut counts the raw defect set in + both modes. + + Failure modes: the interpolated radius is applied to noise-filled pixels + (a wide hole next to the object survives), or not applied at all; the + configured radius is ignored; the boundary is inclusive; the fraction + cut counts the zero-weight orbit. + """ + epochs = _fill_veto_epochs() + assert _surviving(epochs, defect_fill="interpolate") == [ + "2100001-10", "2100002-11" + ] + assert _surviving(epochs) == ["2100001-10"] + assert _surviving( + epochs, defect_fill="interpolate", + epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS + 1, + ) == ["2100001-10"] + assert _surviving(_epochs(), defect_fill="interpolate") == [ + "2100001-10", "2100002-11" + ] + + +def test_process_threads_the_defect_fill(tmp_path, monkeypatch): + """Ngmix hands DEFECT_FILL to both the epoch cuts and metacal: under + interpolation the column at the interpolated-defect radius survives and + metacal is asked to interpolate; under noise it is vetoed. + + Failure mode: the option is dropped on the way to either, so the tile + silently runs the noise fill or the noise cuts. + """ + epochs = {k: v for k, v in _fill_veto_epochs().items() + if k in ("2100001-10", "2100002-11")} + vignet, tile_cat, psf_obj, _ = _fake_inputs(epochs) + vignet.psf_vign_cat = {"1": psf_obj} + vignet.close = lambda: None + tile_cat.obj_id = np.array([1]) + tile_cat.flux = None + monkeypatch.setattr(ngmix_module, "Tile_cat", lambda *a, **k: tile_cat) + for method in ("compile_results", "save_results", "log_mean_ellipticity"): + monkeypatch.setattr(Ngmix, method, lambda *_a, **_k: None) + calls = [] + + def record(stamp, *_args, **kwargs): + calls.append((len(stamp.gals), kwargs.get("defect_fill"))) + raise RuntimeError("metacal is not under test") + + monkeypatch.setattr(ngmix_module, "do_ngmix_metacal", record) + paths = [tmp_path / f"{name}.sqlite" for name in + ("gal", "psf", "weight", "flag", "headers")] + for path in paths: + SqliteDict(str(path)).close() + for fill in ("interpolate", "noise"): + ngmix = Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:4]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), + _RecordingLogger(), bkg_sub=False, defect_fill=fill, + ) + ngmix._vignet_cat.close() + ngmix._vignet_cat = vignet + ngmix.process() + assert calls == [(2, "interpolate"), (1, "noise")] + + +def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): + paths = [tmp_path / f"{name}.sqlite" for name in + ("gal", "psf", "weight", "flag", "headers")] + for path in paths: + SqliteDict(str(path)).close() + with pytest.raises(ValueError, match="DEFECT_FILL"): + Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:4]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), + _RecordingLogger(), bkg_sub=False, defect_fill="interp", + ) diff --git a/tests/science/test_defect_interpolation.py b/tests/science/test_defect_interpolation.py new file mode 100644 index 000000000..d36be39b0 --- /dev/null +++ b/tests/science/test_defect_interpolation.py @@ -0,0 +1,119 @@ +"""Shear recovery for the defects kept under ``DEFECT_FILL = interpolate``. + +Physics invariant: every defect the veto keeps leaves both additive terms +|c1|, |c2| < 5e-4 and both diagonal multiplicative terms |m11|, |m22| < 1%, +from the full 2x2 response matrix. Interpolated defects (columns, full and +finite 3-px bleeds, single pixels) are checked at +``EPOCH_INTERPOLATED_DEFECT_RADIUS`` and two pixels beyond it, on galaxies +with half-light radius 0.3" and 0.5" through a 0.7" PSF, round and with +ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" PSF. Defects +too wide to interpolate are noise-filled, and are checked at +``EPOCH_CENTRAL_DEFECT_RADIUS``. The cases sit at the radii themselves, so +lowering either below its calibrated value turns this red. + +Positive control: a 3-px bleed three pixels inside the interpolated-defect +radius breaks the bound. +""" + +import json + +import numpy as np +import pytest + +from shapepipe.modules.ngmix_package.defect_interpolation import ( + interpolable_defects, +) +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, + central_defect_vetoes, + defect_mask, +) +from tests.helpers.defect_response import defect_response + +N = 51 +CENTRE = N // 2 +RI = int(np.ceil(EPOCH_INTERPOLATED_DEFECT_RADIUS)) +RN = int(np.ceil(EPOCH_CENTRAL_DEFECT_RADIUS)) +ROUND = (0.0, 0.0) +ELLIPTICAL = (0.05, 0.02) +SEEDS = range(6) +INTERPOLATE = {"defect_fill": "interpolate"} + + +def geometry(kind, distance): + """A detector defect whose nearest pixel is ``distance`` px from the + stamp centre.""" + bad = np.zeros((N, N), dtype=bool) + near = CENTRE + distance + if kind == "pixel": + bad[CENTRE, near] = True + elif kind == "column": + bad[:, near] = True + elif kind == "bleed": + bad[:, near:near + 3] = True + elif kind == "finite_bleed": + bad[CENTRE - 5:CENTRE + 6, near:near + 3] = True + elif kind == "wide_bleed": + bad[:, near:near + 5] = True + elif kind == "edge": + bad[:, near:] = True + return bad + + +INTERPOLATED = ("column", "bleed", "finite_bleed", "pixel") +CASES = ( + [(k, d, h, 0.7, ROUND) for k in INTERPOLATED + for d in (RI, RI + 2) for h in (0.3, 0.5)] + + [(k, RI, 0.5, 0.7, ELLIPTICAL) for k in INTERPOLATED] + + [(k, RI, 0.7, 0.9, psf) for k in ("bleed", "finite_bleed") + for psf in (ROUND, ELLIPTICAL)] + + [(k, RN, h, 0.7, ROUND) for k in ("wide_bleed", "edge") + for h in (0.3, 0.5)] +) + + +def case_id(case): + kind, distance, hlr, fwhm, psf_shear = case + psf = "elliptical" if any(psf_shear) else "round" + return f"{kind}-{distance}px-hlr{hlr}-psf{fwhm}-{psf}" + + +def recover(bad, hlr, fwhm, psf_shear, tmp_path): + result = defect_response(bad, hlr=hlr, psf=fwhm, seeds=SEEDS, + psf_shear=psf_shear, options=INTERPOLATE) + (tmp_path / "recovery.json").write_text(json.dumps(result, indent=2)) + return np.abs(result["m"]).max(), np.abs(result["c"]).max(), result + + +@pytest.mark.parametrize("kind,distance,hlr,fwhm,psf_shear", CASES, + ids=[case_id(c) for c in CASES]) +def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, fwhm, + psf_shear, tmp_path): + """Failure modes: a veto radius is below its calibrated value; the + interpolated pixels' quarter-turn orbit keeps its weight (a one-sided + hole in the likelihood); the fill mask is symmetrized; a wide hole is + interpolated; raw defect values leak into metacal.""" + bad = geometry(kind, distance) + masked = defect_mask(np.ones((N, N)), bad.astype(np.int32)) + interpolated = interpolable_defects(masked) + assert interpolated.any() == (kind in INTERPOLATED) + assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT + assert not central_defect_vetoes(masked, EPOCH_CENTRAL_DEFECT_RADIUS, + "interpolate") + m, c, result = recover(bad, hlr, fwhm, psf_shear, tmp_path) + assert m < 0.01, result + assert c < 5e-4, result + + +def test_vetoed_bleed_breaks_the_bound(tmp_path): + """Positive control: a 3-px bleed three pixels inside the + interpolated-defect radius, on the 0.3" galaxy, gives |c| > 1e-3 and + |m| > 1.5%. The veto drops it.""" + bad = geometry("bleed", RI - 3) + assert central_defect_vetoes(bad, EPOCH_CENTRAL_DEFECT_RADIUS, + "interpolate") + m, c, result = recover(bad, 0.3, 0.7, ROUND, tmp_path) + assert c > 1e-3, result + assert m > 0.015, result From 1bb6437afb0286ec6a084758e0549603f53b5b0a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:22:49 +0200 Subject: [PATCH 06/44] feat(ngmix): DEFECT_FILL = interpolate, with a fill-dependent central veto DEFECT_FILL = interpolate fills short defect runs (at most 3 px along a row or column, with clean pixels at both ends: columns, 3-px bleeds, single pixels) with a Clough-Tocher interpolant of the clean pixels within 4 px. The interpolant is averaged over the four quarter turns of the stamp, and each turn's triangulation serves the science image and the metacal noise image alike. Wider holes and runs reaching the stamp border keep the noise fill. noise stays the default and is unchanged. Interpolation restores the object's light, so the remaining bias is the hole in the likelihood: a one-sided zero-weight column 8 px from a 0.5" galaxy still gives c1 = -1.3e-3. The weight is therefore also zero on the quarter-turn orbit of the interpolated pixels (their light stays), which brings that column to c1 = +2e-6. Symmetrizing the fill instead would trade true light for interpolated light over four times the area. Each defect pixel is vetoed at the radius of its own fill: EPOCH_INTERPOLATED_DEFECT_RADIUS (default 7 px) for interpolated pixels and EPOCH_CENTRAL_DEFECT_RADIUS (10 px) for noise-filled ones. 7 px is the smallest radius at which interpolated columns, 3-px bleeds and single pixels give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 on 0.3", 0.5" and 0.7" galaxies, round and elliptical PSFs; it does not scale with galaxy size. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- .../ngmix_package/defect_interpolation.py | 168 ++++++++++++++ src/shapepipe/modules/ngmix_package/ngmix.py | 208 +++++++++++++++--- src/shapepipe/modules/ngmix_runner.py | 28 ++- 3 files changed, 375 insertions(+), 29 deletions(-) create mode 100644 src/shapepipe/modules/ngmix_package/defect_interpolation.py diff --git a/src/shapepipe/modules/ngmix_package/defect_interpolation.py b/src/shapepipe/modules/ngmix_package/defect_interpolation.py new file mode 100644 index 000000000..713e326f7 --- /dev/null +++ b/src/shapepipe/modules/ngmix_package/defect_interpolation.py @@ -0,0 +1,168 @@ +"""DEFECT INTERPOLATION. + +Clough-Tocher interpolation of short defect runs before metacal, selected +with ``DEFECT_FILL = interpolate`` (see +:func:`shapepipe.modules.ngmix_package.ngmix.prepare_ngmix_weights`). + +""" + +import numpy as np +from scipy.interpolate import CloughTocher2DInterpolator +from scipy.ndimage import binary_dilation, label +from scipy.spatial import QhullError + +# Longest row or column run of defect pixels that is interpolated. +MAX_INTERPOLATED_RUN = 3 + +# Clean pixels within this Chebyshev distance (pixels) of the interpolated +# pixels support the interpolant. +SUPPORT_RADIUS = 4 + +_ROW_RUNS = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]]) + + +def _short_row_runs(defect, max_run): + """Pixels in row runs of at most ``max_run`` defects that stop short of + both stamp borders.""" + labels, n_runs = label(defect, structure=_ROW_RUNS) + if n_runs == 0: + return np.zeros_like(defect) + short = np.bincount(labels.ravel(), minlength=n_runs + 1) <= max_run + short[0] = False + short[labels[:, 0]] = False + short[labels[:, -1]] = False + return short[labels] + + +def interpolable_defects(defect, max_run=MAX_INTERPOLATED_RUN): + """Defect pixels that ``DEFECT_FILL = interpolate`` interpolates. + + @sc [decision:shape_measurement.defect_fill] interpolable-defects + A defect pixel is interpolated when its row or its column run of defect + pixels is at most ``max_run`` (3) long and has clean pixels at both + ends. That covers columns, 3-px bleeds and isolated pixels, the widths + whose shear recovery is calibrated. Wider holes, and runs that reach the + stamp border (edge bands, corners), have clean light on one side only; + they are noise-filled and vetoed at the noise-fill radius (see + :func:`~shapepipe.modules.ngmix_package.ngmix.central_defect_vetoes`). + The rule reads only the mask and commutes with quarter turns of the + stamp. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp. + max_run : int, optional + Longest interpolated run; the default is ``MAX_INTERPOLATED_RUN``. + + Returns + ------- + numpy.ndarray of bool + ``True`` on the defect pixels to interpolate. + """ + defect = np.asarray(defect, dtype=bool) + return ( + _short_row_runs(defect, max_run) + | _short_row_runs(defect.T, max_run).T + ) + + +def fourfold(mask): + """Union of a square stamp mask with its quarter turns. + + Parameters + ---------- + mask : numpy.ndarray of bool + Square mask. + + Returns + ------- + numpy.ndarray of bool + ``mask`` ORed with its rotations by 90, 180 and 270 degrees about + the stamp centre. + + Raises + ------ + ValueError + If ``mask`` is not square. + """ + mask = np.asarray(mask, dtype=bool) + if mask.ndim != 2 or mask.shape[0] != mask.shape[1]: + raise ValueError( + f"A quarter-turn orbit needs a square stamp, not {mask.shape}" + ) + return mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) + + +def _interpolate_once(planes, defect, target): + """One Clough-Tocher interpolant of every plane at ``target``; NaN + elsewhere and where the support cannot reach.""" + out = np.full(planes.shape, np.nan) + support = binary_dilation( + target, structure=np.ones((3, 3), dtype=bool), + iterations=SUPPORT_RADIUS, + ) & ~defect + points = np.argwhere(support).astype(float) + if len(points) < 3: + return out + query = np.argwhere(target) + try: + interpolant = CloughTocher2DInterpolator( + points, planes[:, support].T, fill_value=np.nan, + ) + except QhullError: + return out + out[:, query[:, 0], query[:, 1]] = interpolant(query.astype(float)).T + return out + + +def interpolate_defects(planes, defect, target): + """Replace the ``target`` pixels of every plane by a Clough-Tocher + interpolant of the clean pixels around them. + + @sc [decision:shape_measurement.defect_fill] shared-rotation-averaged-interpolant + The support is the clean pixels within ``SUPPORT_RADIUS`` (4 px) of the + target; no defect pixel enters it, so defect values are never read. For + each quarter turn of the stamp, one Delaunay triangulation of the support + serves every plane, so the science image and the metacal noise image + see the same linear operator and fixnoise mirrors the science image's + interpolated noise. A regular grid's triangulation has degenerate + diagonals, so one orientation has a preferred direction; averaging the + four quarter-turned operators makes the fill commute with rotations of + the stamp. That removes up to 6e-5 of c2 for a column or 3-px bleed + 6 px from the object. + + Parameters + ---------- + planes : array_like + Stamp planes, shape ``(n, ny, nx)``. + defect : numpy.ndarray of bool + Every defect pixel, shape ``(ny, nx)``; none of them supports the + interpolant. + target : numpy.ndarray of bool + Defect pixels to interpolate (:func:`interpolable_defects`). + + Returns + ------- + numpy.ndarray + A copy of ``planes`` with ``target`` pixels interpolated; NaN at a + target pixel whose support is degenerate in some orientation. + """ + planes = np.asarray(planes, dtype=float) + defect = np.asarray(defect, dtype=bool) + target = np.asarray(target, dtype=bool) + out = planes.copy() + if not target.any(): + return out + turns = [ + np.rot90( + _interpolate_once( + np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), + np.rot90(target, k), + ), + -k, axes=(1, 2), + )[:, target] + for k in range(4) + ] + out[:, target] = np.mean(turns, axis=0) + return out diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index a19d2a83e..acb425835 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -23,11 +23,20 @@ from scipy.spatial import cKDTree from sqlitedict import SqliteDict +from shapepipe.modules.ngmix_package.defect_interpolation import ( + fourfold, + interpolable_defects, + interpolate_defects, +) from shapepipe.pipeline import file_io # Neighbour treatments selectable with the BLEND_HANDLING option. BLEND_HANDLINGS = ("none", "uberseg") +# Defect fills selectable with the DEFECT_FILL option (see +# :func:`prepare_ngmix_weights`). +DEFECT_FILLS = ("noise", "interpolate") + # Default of the EPOCH_MASKED_FRACTION_CUT option: an epoch is dropped when # more than this fraction of its stamp is in :func:`defect_mask` (see # :func:`prepare_postage_stamps`). @@ -38,6 +47,11 @@ # stamp centre (see :func:`has_central_defect`). EPOCH_CENTRAL_DEFECT_RADIUS = 10 +# Default of the EPOCH_INTERPOLATED_DEFECT_RADIUS option (pixels): under +# DEFECT_FILL = interpolate, the veto radius for interpolated defect pixels +# (see :func:`central_defect_vetoes`). +EPOCH_INTERPOLATED_DEFECT_RADIUS = 7 + METACAL_TYPES = ('noshear', '1p', '1m', '2p', '2m') # Noise budget for the PSF observation's flat weight map (psf_wt = @@ -444,8 +458,7 @@ class Ngmix(object): Neighbour treatment. ``"none"`` (default) leaves neighbour pixels untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect - pixels are noise-filled under both (see - :func:`prepare_ngmix_weights`). + pixels are filled under both (see :func:`prepare_ngmix_weights`). seg_cat_path : str, optional Path to the coadd-frame segmentation VIGNET catalogue (see :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. @@ -460,6 +473,14 @@ class Ngmix(object): Drop an epoch when more than this fraction of its stamp is masked (see :func:`prepare_postage_stamps`); the default is ``EPOCH_MASKED_FRACTION_CUT``. + defect_fill : {"noise", "interpolate"}, optional + How defect pixels are filled before metacal (see + :func:`prepare_ngmix_weights`); the default is ``"noise"``. + epoch_interpolated_defect_radius : float, optional + Under ``defect_fill="interpolate"``, drop an epoch when an + interpolated defect pixel lies closer than this many pixels to the + stamp centre (see :func:`central_defect_vetoes`); the default is + ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. Notes ----- @@ -471,8 +492,8 @@ class Ngmix(object): IndexError If the length of the input file list is incorrect ValueError - If ``blend_handling`` is unknown, or ``"uberseg"`` is selected without - ``seg_cat_path``. + If ``blend_handling`` or ``defect_fill`` is unknown, or ``"uberseg"`` + is selected without ``seg_cat_path``. """ @@ -496,6 +517,8 @@ def __init__( metacal_psf="fitgauss", epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, + defect_fill="noise", + epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -514,6 +537,11 @@ def __init__( f"Unknown BLEND_HANDLING '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) + if defect_fill not in DEFECT_FILLS: + raise ValueError( + f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" + + f" {DEFECT_FILLS}" + ) # Fail fast at construction (not deep in the per-epoch loop) when # uberseg is requested without its segmentation input (shapepipe#776). @@ -567,6 +595,10 @@ def __init__( self._metacal_psf = metacal_psf self._epoch_central_defect_radius = epoch_central_defect_radius self._epoch_masked_fraction_cut = epoch_masked_fraction_cut + self._defect_fill = defect_fill + self._epoch_interpolated_defect_radius = ( + epoch_interpolated_defect_radius + ) self._w_log = w_log @@ -1046,6 +1078,10 @@ def process(self): gal_obj, epoch_central_defect_radius=self._epoch_central_defect_radius, epoch_masked_fraction_cut=self._epoch_masked_fraction_cut, + defect_fill=self._defect_fill, + epoch_interpolated_defect_radius=( + self._epoch_interpolated_defect_radius + ), ) epoch_cuts.update(stamp.epoch_cuts) @@ -1092,6 +1128,7 @@ def process(self): object_number=obj_id, dilate_neighbour=self._dilate_neighbour, metacal_psf=self._metacal_psf, + defect_fill=self._defect_fill, ) except Exception as ee: self._w_log.info( @@ -1193,6 +1230,8 @@ def prepare_postage_stamps( gal_obj=None, epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, + defect_fill="noise", + epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, ): """Gather one object's epoch stamps, dropping epochs its defects spoil. @@ -1200,8 +1239,9 @@ def prepare_postage_stamps( An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS pixels, the set that :func:`prepare_ngmix_weights` zero-weights and - noise-fills. Counting flags alone would keep epochs whose filled area - exceeds the cut. The default is 1/3; 10%, the DES Y3 and Y6 value, is + fills. Counting flags alone would keep epochs whose filled area exceeds + the cut. The zero-weight orbit of interpolated pixels keeps its light and + is not counted. The default is 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. Parameters @@ -1225,6 +1265,14 @@ def prepare_postage_stamps( epoch_masked_fraction_cut : float, optional Drop an epoch with more than this fraction of its stamp in :func:`defect_mask`. The default is ``EPOCH_MASKED_FRACTION_CUT``. + defect_fill : {"noise", "interpolate"}, optional + The fill :func:`prepare_ngmix_weights` will apply; it sets the veto + radius of each defect pixel (:func:`central_defect_vetoes`). The + default is ``"noise"``. + epoch_interpolated_defect_radius : float, optional + Veto radius for interpolated defect pixels under + ``defect_fill="interpolate"``. The default is + ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. Returns ------- @@ -1307,14 +1355,17 @@ def prepare_postage_stamps( else None ) # Drop the epoch when too much of it would be zero-weighted and - # noise-filled (epoch-cut-on-defect-mask), or when a filled pixel - # would sit near the object (epoch-central-defect-veto). + # filled (epoch-cut-on-defect-mask), or when a filled pixel would sit + # near the object (veto-radius-follows-the-fill). masked = defect_mask(weight_vign, flag_vign, bkg_rms_vign) stamp.epoch_cuts["considered"] += 1 if masked.mean() > epoch_masked_fraction_cut: stamp.epoch_cuts["masked_fraction"] += 1 continue - if has_central_defect(masked, epoch_central_defect_radius): + if central_defect_vetoes( + masked, epoch_central_defect_radius, defect_fill, + epoch_interpolated_defect_radius, + ): stamp.epoch_cuts["central_veto"] += 1 continue @@ -1734,6 +1785,59 @@ def has_central_defect(defect, radius): return bool(np.any(distance < radius)) +def central_defect_vetoes( + defect, radius, defect_fill="noise", + interpolated_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, +): + """Whether the central-defect veto drops an epoch under ``defect_fill``. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.defect_fill] veto-radius-follows-the-fill + Each defect pixel is vetoed (:func:`has_central_defect`) at the radius + calibrated for the operator that fills it. Under ``"noise"`` every + defect keeps ``radius`` (``EPOCH_CENTRAL_DEFECT_RADIUS``). Under + ``"interpolate"`` the pixels :func:`interpolable_defects` selects are + interpolated and vetoed at ``interpolated_radius`` + (``EPOCH_INTERPOLATED_DEFECT_RADIUS``); the others are noise-filled and + keep ``radius``. 7 px is the smallest interpolated radius at which + columns, full and finite 3-px bleeds and single pixels give + |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 (full response matrix) on + galaxies with half-light radius 0.3" and 0.5" through a 0.7" PSF, round + or with ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" + PSF. There the worst case is c1 = 3.6e-4 (3-px bleed, 0.7" galaxy, + elliptical PSF); on the 0.3" and 0.5" galaxies |c| <= 5e-5 and + |m| <= 0.21%. At 6 px those two still pass (|c| <= 3.1e-4), but a 3-px + bleed on the 0.7" galaxy gives c1 = 7.5e-4; at 5 px a 3-px bleed on the + 0.5" galaxy gives 9.5e-4. The radius does not scale with galaxy size: + interpolation needs one pixel more for the 0.7" galaxy (noise fill needs + four, 10 to 14 px), and a size-dependent veto would select on measured + size, which responds to shear. Guarded by + ``tests/science/test_defect_interpolation.py``. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp (:func:`defect_mask`). + radius : float + Veto radius in pixels for noise-filled defect pixels. + defect_fill : {"noise", "interpolate"}, optional + The fill the epoch will get; the default is ``"noise"``. + interpolated_radius : float, optional + Veto radius in pixels for interpolated defect pixels; the default is + ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. + + Returns + ------- + bool + ``True`` if the epoch should be dropped. + """ + if defect_fill == "interpolate": + interpolated = interpolable_defects(defect) + return has_central_defect( + interpolated, interpolated_radius + ) or has_central_defect(defect & ~interpolated, radius) + return has_central_defect(defect, radius) + + def fill_defects(image, defect, noise): """Replace an image's defect pixels by a noise realisation. @@ -1766,24 +1870,42 @@ def fill_defects(image, defect, noise): def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="none", seg=None, object_number=None, - dilate_neighbour=0, + dilate_neighbour=0, defect_fill="noise", ): """Build one epoch's image, weight map and noise image for ngmix. Defect pixels (:func:`defect_mask`: flagged, zero-weight or invalid-RMS - pixels) get weight 0 and are replaced by an independent noise - realisation at their background RMS (:func:`fill_defects`), under either - ``blend_handling``. ``blend_handling`` decides only the neighbour side. + pixels) get weight 0 and are filled under either ``blend_handling``: + by an independent noise realisation at their background RMS + (:func:`fill_defects`), or under ``defect_fill="interpolate"``, for + short defect runs, by an interpolant of the clean pixels around them. + ``blend_handling`` decides only the neighbour side. @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-neighbours-raw - Every pixel of ``defect_mask`` is zero-weighted and noise-filled whatever - ``blend_handling`` is, and the filled set equals the zero-weight defect - set. ``blend_handling`` acts on the neighbour side only: uberseg zeroes + Every pixel of ``defect_mask`` is zero-weighted and filled whatever + ``blend_handling`` is, and the filled set equals the defect set. + ``blend_handling`` acts on the neighbour side only: uberseg zeroes the weight of pixels nearer a neighbour's footprint and leaves their image values raw, because that light is real sky that metacal shears along with the target; filling it with noise would cut the target along - an unsheared edge. Neighbour pixels are never noise-filled. The noise - image covers the whole stamp and is independent of both. + an unsheared edge. Neighbour pixels are never filled. The noise image + covers the whole stamp and is independent of both. + + @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defect-weight-orbit + Under ``defect_fill="interpolate"`` the short defect runs + (:func:`interpolable_defects`) take the interpolant of the clean pixels + around them (:func:`interpolate_defects`) in the image and in the noise + image alike, and the other defects are noise-filled. Interpolation + restores the object's light, which leaves the hole in the likelihood: + ngmix fits a Gaussian to a non-Gaussian profile, and a one-sided + zero-weight hole pulls that fit. Once interpolated, a column 8 px from a + galaxy with half-light radius 0.5" through a 0.7" PSF still gives + c1 = -1.3e-3. The weight is therefore also zero on the quarter-turn + orbit of the interpolated pixels, whose light stays, which cancels that + spin-2 term: c1 = +2e-6 for the same column. Only the weights are + symmetrized, not the fill: interpolating the orbit as well would trade + true light for interpolated light over four times the area, raising m11 + for a 3-px bleed 6 px from the same galaxy from +0.19% to +0.89%. Parameters ---------- @@ -1816,23 +1938,31 @@ def prepare_ngmix_weights( dilate_neighbour : int, optional Neighbour-mask dilation iterations, passed to :func:`uberseg_weight` under ``blend_handling="uberseg"``; ignored otherwise. + defect_fill : {"noise", "interpolate"}, optional + ``"noise"`` (default) noise-fills every defect; ``"interpolate"`` + interpolates the short defect runs and noise-fills the rest. Returns ------- numpy.ndarray - Galaxy image with defect pixels noise-filled. + Galaxy image with defect pixels filled. numpy.ndarray Inverse-variance weight map for ngmix. numpy.ndarray Noise image: an independent realisation over the whole stamp, for - metacal's ``fixnoise``. + metacal's ``fixnoise``, interpolated where the galaxy image is. Raises ------ ValueError - If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or - ``object_number``. + If ``blend_handling`` or ``defect_fill`` is unknown, or + ``"uberseg"`` lacks ``seg`` or ``object_number``. """ + if defect_fill not in DEFECT_FILLS: + raise ValueError( + f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" + + f" {DEFECT_FILLS}" + ) if blend_handling not in BLEND_HANDLINGS: raise ValueError( f"Unknown blend_handling '{blend_handling}'; expected one of" @@ -1848,19 +1978,29 @@ def prepare_ngmix_weights( defect = defect_mask(weight, flag, bkg_rms) clean = ~defect + interpolated = ( + interpolable_defects(defect) + if defect_fill == "interpolate" + else np.zeros_like(defect) + ) + # The quarter-turn orbit of the interpolated pixels carries no weight + # (interpolated-defect-weight-orbit). + weighted = ( + ~(defect | fourfold(interpolated)) if interpolated.any() else clean + ) if bkg_rms is None: sig_noise = sigma_mad(gal) # Guard the degenerate constant stamp (sigma_mad == 0): 0 * inf # would otherwise put NaN in a fully-masked weight map. weight_map = ( - clean.astype(float) / sig_noise ** 2 + weighted.astype(float) / sig_noise ** 2 if sig_noise > 0 else np.zeros_like(gal, dtype=float) ) else: weight_map = np.zeros_like(gal, dtype=float) - weight_map[clean] = 1.0 / bkg_rms[clean] ** 2 + weight_map[weighted] = 1.0 / bkg_rms[weighted] ** 2 # Per-pixel noise sigma for the realisations below: metacal's # fixnoise bookkeeping (1/w + 1/w_noise) assumes the noise image # is a faithful realisation of the per-pixel variance the weights @@ -1884,6 +2024,13 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_filled = fill_defects(gal, defect, noise_img_gal) + if interpolated.any(): + # One operator for the image and the noise image; a pixel whose + # support is degenerate keeps its noise fill. + filled = interpolate_defects([gal, noise_img], defect, interpolated) + done = interpolated & np.all(np.isfinite(filled), axis=0) + gal_filled[done] = filled[0][done] + noise_img = np.where(done, filled[1], noise_img) if blend_handling == "uberseg": weight_map = uberseg_weight( @@ -1897,7 +2044,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="none", seg=None, object_number=None, - dilate_neighbour=0, + dilate_neighbour=0, defect_fill="noise", ): """Build an ngmix Observation for a single galaxy epoch. @@ -1951,6 +2098,9 @@ def make_ngmix_observation( dilate_neighbour : int, optional Neighbour-mask dilation iterations passed through to :func:`prepare_ngmix_weights` under ``blend_handling="uberseg"``. + defect_fill : {"noise", "interpolate"}, optional + Defect fill passed through to :func:`prepare_ngmix_weights`; the + default is ``"noise"``. Returns ------- @@ -1978,7 +2128,7 @@ def make_ngmix_observation( gal_masked, weight_map, noise_img = prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, - dilate_neighbour=dilate_neighbour, + dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, ) if centroid_source == "hsm": @@ -2204,7 +2354,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", blend_handling="none", object_number=None, dilate_neighbour=0, - metacal_psf="fitgauss", + metacal_psf="fitgauss", defect_fill="noise", ): """Do Ngmix Metacal. @@ -2251,6 +2401,9 @@ def do_ngmix_metacal( round PSF that metacal reconvolves with after shearing, so it moves the metacal *response* (and therefore the recovered shear) but never reaches the deconvolution, which is by the PSF image. + defect_fill : {"noise", "interpolate"}, optional + Defect fill passed through to :func:`make_ngmix_observation`; the + default is ``"noise"``. Returns ------- @@ -2283,6 +2436,7 @@ def do_ngmix_metacal( seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, object_number=object_number, dilate_neighbour=dilate_neighbour, + defect_fill=defect_fill, ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 605262395..158e14ef8 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -13,6 +13,7 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, EPOCH_MASKED_FRACTION_CUT, Ngmix, ) @@ -133,8 +134,8 @@ def ngmix_runner( # weighted and untouched; "uberseg" zeroes the weight of every pixel # closer to a neighbour than to the central object, from the segmentation # map, and leaves its image raw. Defect pixels (flagged, zero-weight or - # invalid-RMS) are zero-weighted and noise-filled under both; see - # prepare_ngmix_weights. + # invalid-RMS) are zero-weighted and filled under both (DEFECT_FILL + # below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: @@ -166,6 +167,27 @@ def ngmix_runner( else: epoch_masked_fraction_cut = EPOCH_MASKED_FRACTION_CUT + # DEFECT_FILL (optional): "noise" (default) noise-fills every defect; + # "interpolate" interpolates short defect runs (at most 3 px along a row + # or column) from the clean pixels around them and noise-fills the rest. + if config.has_option(module_config_sec, "DEFECT_FILL"): + defect_fill = config.get(module_config_sec, "DEFECT_FILL") + else: + defect_fill = "noise" + + # EPOCH_INTERPOLATED_DEFECT_RADIUS (optional, pixels): under + # DEFECT_FILL = interpolate, drop an epoch when an interpolated defect + # pixel lies closer than this to the stamp centre; noise-filled defect + # pixels keep EPOCH_CENTRAL_DEFECT_RADIUS. + if config.has_option( + module_config_sec, "EPOCH_INTERPOLATED_DEFECT_RADIUS" + ): + epoch_interpolated_defect_radius = config.getfloat( + module_config_sec, "EPOCH_INTERPOLATED_DEFECT_RADIUS" + ) + else: + epoch_interpolated_defect_radius = EPOCH_INTERPOLATED_DEFECT_RADIUS + # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -223,6 +245,8 @@ def ngmix_runner( metacal_psf=metacal_psf, epoch_central_defect_radius=epoch_central_defect_radius, epoch_masked_fraction_cut=epoch_masked_fraction_cut, + defect_fill=defect_fill, + epoch_interpolated_defect_radius=epoch_interpolated_defect_radius, ) # Process ngmix shape measurement and metacalibration From b9d92a9643538346289e16145834d0ef8d6d35f0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:14:13 +0200 Subject: [PATCH 07/44] ngmix: restore the BLEND_HANDLING name noisefill The neighbour option keeps develop's name, noisefill, in the code, the runner config key, tests, astra.yaml and the committed universe; the name none is dropped. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 8 +++--- src/shapepipe/modules/ngmix_package/ngmix.py | 30 ++++++++++---------- src/shapepipe/modules/ngmix_runner.py | 4 +-- tests/module/test_ngmix.py | 2 +- tests/module/test_ngmix_defect_fill.py | 4 +-- tests/module/test_ngmix_uberseg.py | 11 ++----- universes/committed.yaml | 2 +- 7 files changed, 27 insertions(+), 34 deletions(-) diff --git a/astra.yaml b/astra.yaml index 85ee5f48c..978fe94a7 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1392,7 +1392,7 @@ analyses: rationale: >- Covers only pixels shared with a neighbour; defect_fill does not depend on it: defects are noise-filled under every BLEND_HANDLING. - none (the default; the committed config + noisefill (the default; the committed config sets no key) leaves neighbours fully weighted and untouched. uberseg zeroes the weight of pixels nearer a neighbour's coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) @@ -1415,10 +1415,10 @@ analyses: instead cut the target's own light along an unsheared boundary, a sharp edge that rings in the FFTs. The recommended comparison arm is uberseg (weight-only), with defect_fill held equal across arms. - default: none + default: noisefill options: - none: - label: No neighbour treatment (BLEND_HANDLING = none) + noisefill: + label: No neighbour treatment (BLEND_HANDLING = noisefill) description: >- Neighbour pixels keep their full weight and image values, so the fit sees all neighbour light, which biases shapes toward diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 9ebb018d3..5bd942913 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -31,7 +31,7 @@ from shapepipe.pipeline import file_io # Neighbour treatments selectable with the BLEND_HANDLING option. -BLEND_HANDLINGS = ("none", "uberseg") +BLEND_HANDLINGS = ("noisefill", "uberseg") # Defect fills selectable with the DEFECT_FILL option (see # :func:`prepare_ngmix_weights`). @@ -550,7 +550,7 @@ def get_data(self, cat_path): # Coadd-frame SExtractor segmentation stamp (integer labels), one per # object and row-aligned to the tile catalogue, overlaid unchanged on # every epoch for uberseg neighbour masking (shapepipe#776). None -> - # uberseg unavailable; ``"none"`` does not read it. + # uberseg unavailable; ``"noisefill"`` does not read it. self.seg = None if self.seg_cat_path: seg_cat = file_io.FITSCatalogue( @@ -605,7 +605,7 @@ def __init__( self.flags = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend - # handling; empty under the default "none". All epochs carry the + # handling; empty under the default "noisefill". All epochs carry the # SAME coadd-frame seg stamp (shapepipe#776: one coadd seg per object, # no per-epoch reprojection), each MegaCam-flipped to match its galaxy # stamp so the overlay stays registered. @@ -753,8 +753,8 @@ class Ngmix(object): propagated on the vignette. ``"hsm"`` re-centers on the adaptive-moment centroid measured from the stamp pixels. See :func:`make_ngmix_observation`. - blend_handling : {"none", "uberseg"}, optional - Neighbour treatment. ``"none"`` (default) leaves neighbour pixels + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment. ``"noisefill"`` (default) leaves neighbour pixels untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect pixels are filled under both (see :func:`prepare_ngmix_weights`). @@ -810,7 +810,7 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="wcs", - blend_handling="none", + blend_handling="noisefill", seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", @@ -2144,7 +2144,7 @@ def fill_defects(image, defect, noise): def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, - blend_handling="none", seg=None, object_number=None, + blend_handling="noisefill", seg=None, object_number=None, dilate_neighbour=0, defect_fill="noise", ): """Build one epoch's image, weight map and noise image for ngmix. @@ -2198,8 +2198,8 @@ def prepare_ngmix_weights( ``1 / bkg_rms**2`` as the ngmix inverse variance, and non-finite or non-positive values mark defects. Otherwise every clean pixel gets ``1 / sigma_mad(gal)**2``. - blend_handling : {"none", "uberseg"}, optional - Neighbour treatment. ``"none"`` (default) leaves neighbour + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment. ``"noisefill"`` (default) leaves neighbour pixels weighted and untouched. ``"uberseg"`` zeroes the weight of every pixel closer to a neighbour's segmentation footprint than to the central object's and keeps its raw image value (see @@ -2319,7 +2319,7 @@ def prepare_ngmix_weights( def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, - blend_handling="none", seg=None, object_number=None, + blend_handling="noisefill", seg=None, object_number=None, dilate_neighbour=0, defect_fill="noise", ): """Build an ngmix Observation for a single galaxy epoch. @@ -2362,9 +2362,9 @@ def make_ngmix_observation( Sub-pixel ``[row, col]`` coadd-centroid offset propagated from the stamp extractor. Required for ``centroid_source="wcs"`` (ignored for ``"hsm"``). - blend_handling : {"none", "uberseg"}, optional + blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to :func:`prepare_ngmix_weights`; - the default ``"none"`` leaves neighbour pixels untouched. + the default ``"noisefill"`` leaves neighbour pixels untouched. seg : numpy.ndarray, optional Segmentation map on the stamp grid. Required for ``blend_handling="uberseg"`` (ignored otherwise). @@ -2635,7 +2635,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", - blend_handling="none", object_number=None, dilate_neighbour=0, + blend_handling="noisefill", object_number=None, dilate_neighbour=0, metacal_psf="fitgauss", defect_fill="noise", ): """Do Ngmix Metacal. @@ -2659,9 +2659,9 @@ def do_ngmix_metacal( coadd-centroid offset in ``stamp.offsets``, propagated from the stamp extractor); ``"hsm"`` uses the adaptive-moment centroid from the stamp pixels — see that function. - blend_handling : {"none", "uberseg"}, optional + blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to - :func:`make_ngmix_observation`; the default ``"none"`` leaves + :func:`make_ngmix_observation`; the default ``"noisefill"`` leaves neighbour pixels untouched. ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index d350ba6fb..166e8cae5 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -144,7 +144,7 @@ def ngmix_runner( else: centroid_source = "wcs" - # Neighbour treatment: "none" (default) leaves neighbour pixels + # Neighbour treatment: "noisefill" (default) leaves neighbour pixels # weighted and untouched; "uberseg" zeroes the weight of every pixel # closer to a neighbour than to the central object, from the segmentation # map, and leaves its image raw. Defect pixels (flagged, zero-weight or @@ -153,7 +153,7 @@ def ngmix_runner( if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: - blend_handling = "none" + blend_handling = "noisefill" # DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the # uberseg neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index edd54c118..a69aa0e69 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -660,7 +660,7 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags inst._id_obj_min = inst._id_obj_max = -1 inst._bkg_sub = True inst._pixel_scale = .186 - inst._blend_handling = "none" + inst._blend_handling = "noisefill" inst._dilate_neighbour = 1 inst._metacal_psf = "fitgauss" inst._epoch_central_defect_radius = module.EPOCH_CENTRAL_DEFECT_RADIUS diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index ac10057b7..74cb3045b 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -75,7 +75,7 @@ def _uberseg_seg(n): @given( stamp=defect_stamps(), - blend_handling=st.sampled_from(["none", "uberseg"]), + blend_handling=st.sampled_from(["noisefill", "uberseg"]), seed=st.integers(0, 2**31 - 1), ) def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): @@ -572,7 +572,7 @@ def _hot_stamp(): return gal, np.ones((N_STAMP, N_STAMP)), flag -@pytest.mark.parametrize("blend_handling", ["none", "uberseg"]) +@pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) def test_interpolated_fill_and_its_weights(blend_handling): """Short defect runs take the interpolant of the clean image; other defects take noise; the weight is zero on the defects and on the diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 4d6ebb32a..92616fde0 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -239,8 +239,8 @@ def _gal_flag_weight(npix=41, seed=7): return gal, flag, weight -def test_none_ignores_seg_and_dilate_kwargs(): - """Under BLEND_HANDLING = none, passing seg / dilate_neighbour changes +def test_noisefill_ignores_seg_and_dilate_kwargs(): + """Under BLEND_HANDLING = noisefill, passing seg / dilate_neighbour changes nothing: the result matches the plain default call on the same RNG stream.""" gal, flag, weight = _gal_flag_weight() seg, _, _ = two_object_seg(npix=gal.shape[0], sep=12) @@ -406,10 +406,3 @@ def test_runner_missing_seg_vignet_file_raises(tmp_path): _RecordingLogger(), ) - -def test_retired_noisefill_name_is_rejected(): - """Do not silently interpret the retired neighbour option.""" - with pytest.raises(ValueError, match="noisefill"): - prepare_ngmix_weights(np.ones((5, 5)), np.ones((5, 5)), - np.zeros((5, 5)), np.random.RandomState(3), - blend_handling="noisefill") diff --git a/universes/committed.yaml b/universes/committed.yaml index 4e3e02bec..6f2cf8da7 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -52,7 +52,7 @@ analyses: psf_likelihood_noise: psf_noise_1em5 megacam_ccd_flip: megapipe_flip defect_fill: noise - blend_handling: none + blend_handling: noisefill epoch_masked_fraction_cut: one_third central_defect_veto: fixed_radii catalogue_assembly: From 23cc5cb29e1035ac766d4cd1a047216a0548ad6c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:16:34 +0200 Subject: [PATCH 08/44] test(ngmix): specify neighbour markers apart from defects SExtractor's -1e30 markers in the tile VIGNET form their own per-epoch mask (stamp.neighbours): a footprint 3 px from the centre, or covering 45% of the stamp, drops no epoch; noisefill zero-weights and noise-fills exactly the marked pixels, bit for bit as develop does; uberseg leaves them raw and weighted; a flagged column near the centre is still vetoed; and do_ngmix_metacal hands each epoch's mask to make_ngmix_observation. Co-Authored-By: Claude Opus 5.5 --- tests/module/test_ngmix_defect_fill.py | 267 +++++++++++++++++++++++++ 1 file changed, 267 insertions(+) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 74cb3045b..b9e3096c5 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -10,6 +10,9 @@ in :func:`prepare_postage_stamps` act on the same defect set: the masked-fraction cut counts it, and the central-defect veto drops an epoch with a defect near the stamp centre, at a radius set by the defect's fill. +SExtractor's -1e30 neighbour markers in the tile VIGNET are not defects: +noisefill zero-weights and noise-fills them, uberseg ignores them, and the +epoch cuts never count them. """ import re @@ -22,6 +25,7 @@ from astropy.wcs import WCS from hypothesis import given from hypothesis import strategies as st +from modopt.math.stats import sigma_mad from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package import ngmix as ngmix_module @@ -735,3 +739,266 @@ def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), _RecordingLogger(), bkg_sub=False, defect_fill="interp", ) + + +# --- SExtractor's -1e30 neighbour markers are not defects ------------------- +# +# The tile VIGNET carries -1e30 on the footprints of other detections. Every +# epoch shares that tile stamp, so a marker counted as a defect would drop +# every epoch of an object with a neighbour inside the veto radius. The +# markers are their own per-epoch mask (``stamp.neighbours``): noisefill +# zero-weights and noise-fills them, uberseg ignores them, and the epoch cuts +# never read them. + +_MARKER = -1.0e30 +_CENTRE = N_STAMP // 2 +# Flipped (ccd < 18) and unflipped (ccd >= 18) MegaCam CCDs. +_MARKER_EPOCH_NAMES = ["2100001-10", "2100002-20", "2100003-11"] + + +def _tile_with_neighbour(columns_from=_CENTRE + 3, rows=(_CENTRE - 2, + _CENTRE + 3)): + """Tile VIGNET with a -1e30 neighbour footprint whose nearest pixel is + 3 px from the stamp centre. The footprint is off-centre, so the MegaCam + flip moves it.""" + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[rows[0]:rows[1], columns_from:columns_from + 6] = _MARKER + return tile + + +def _marker_stamp(tile, epochs=None, **kwargs): + """Run prepare_postage_stamps on defect-free epochs under ``tile``.""" + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + if epochs is None: + epochs = {name: (clean.copy(), ones) for name in _MARKER_EPOCH_NAMES} + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(epochs) + tile_cat.vign = tile[np.newaxis] + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, **kwargs, + ) + return stamp, epochs, gal_obj + + +def _expected_neighbours(tile, name): + return Ngmix.MegaCamFlip(tile, int(name.split("-")[1])) == _MARKER + + +def test_neighbour_markers_near_the_centre_keep_every_epoch(): + """A neighbour footprint 3 px from the centre, and one covering 45% of + the stamp, drop no epoch: the masked-fraction cut and the central veto + count no marker. + + Failure mode: the markers are written into the flag stamp and counted as + defects, so every epoch (they all share the tile VIGNET) is dropped by + the veto or the fraction cut and the object loses its shape + (neighbour-markers-are-not-defects). + """ + small = _tile_with_neighbour() + large = _tile_with_neighbour(rows=(0, N_STAMP)) + large[:, _CENTRE + 3:] = _MARKER + assert (large == _MARKER).mean() > 1 / 3 + for tile in (small, large): + stamp, _, _ = _marker_stamp(tile) + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + assert stamp.epoch_cuts["considered"] == len(_MARKER_EPOCH_NAMES) + assert stamp.epoch_cuts["masked_fraction"] == 0 + assert stamp.epoch_cuts["central_veto"] == 0 + + +def test_neighbour_markers_are_their_own_per_epoch_mask(): + """``stamp.neighbours`` holds the MegaCam-flipped marker mask of each + surviving epoch, and the flag stamps stay the exposure's own. + + Failure mode: the markers are merged into the flags, or the neighbour + mask is not flipped with its epoch and lands on the wrong pixels. + """ + tile = _tile_with_neighbour() + stamp, epochs, _ = _marker_stamp(tile) + names = {id(v[0]): name for name, v in epochs.items()} + assert len(stamp.neighbours) == len(stamp.flags) + for flag, neighbour in zip(stamp.flags, stamp.neighbours): + name = names[id(flag)] + npt.assert_array_equal(flag, 0) + npt.assert_array_equal(neighbour, _expected_neighbours(tile, name)) + assert not np.array_equal(stamp.neighbours[0], stamp.neighbours[1]) + + +def test_a_flagged_column_near_the_centre_is_still_vetoed(): + """With a neighbour footprint present, an epoch with a genuinely flagged + column 3 px from the centre is still dropped, and only that epoch. + + Failure mode: handling the markers apart also exempts real defects from + the central veto. + """ + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + column = clean.copy() + column[:, _CENTRE - 3] = 1 + epochs = { + "2100001-10": (clean, ones), + "2100002-20": (column, ones), + "2100003-11": (clean.copy(), ones), + } + stamp, _, _ = _marker_stamp(_tile_with_neighbour(), epochs) + names = {id(v[0]): name for name, v in epochs.items()} + assert sorted(names[id(f)] for f in stamp.flags) == [ + "2100001-10", "2100003-11" + ] + assert stamp.epoch_cuts["central_veto"] == 1 + assert stamp.epoch_cuts["masked_fraction"] == 0 + + +def _stamp_epoch_weights(stamp, i, seed, **kwargs): + return prepare_ngmix_weights( + 1.0e3 + stamp.gals[i], stamp.weights[i], stamp.flags[i], + np.random.RandomState(seed), bkg_rms=stamp.bkg_rms[i], + neighbour=stamp.neighbours[i], **kwargs, + ) + + +def test_noisefill_fills_exactly_the_marked_pixels(): + """Under noisefill, a defect-free epoch has zero weight and noise + exactly on the marked neighbour pixels; every other pixel keeps its raw + value and its weight. + + Failure mode: the neighbour markers are dropped with the flags, so + noisefill no longer removes neighbour light (noisefill-fills-markers). + """ + tile = _tile_with_neighbour() + stamp, epochs, _ = _marker_stamp(tile) + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + for i in range(len(stamp.gals)): + gal = 1.0e3 + stamp.gals[i] + gal_out, w_out, _ = _stamp_epoch_weights( + stamp, i, seed=i, blend_handling="noisefill", + ) + neighbour = stamp.neighbours[i] + assert neighbour.any() + npt.assert_array_equal(gal_out != gal, neighbour) + npt.assert_array_equal(w_out == 0.0, neighbour) + assert np.all(np.abs(gal_out[neighbour]) < 10.0) + + +def test_uberseg_leaves_the_marked_pixels_raw(): + """Under uberseg, the markers mask nothing: with a seg map holding only + the central object, a defect-free epoch keeps every pixel raw and + weighted, marked or not. + + Failure mode: the markers reach the defect set or the fill, so uberseg + noise-fills the neighbour's light instead of leaving it to the seg-based + weight (uberseg-ignores-markers). + """ + tile = _tile_with_neighbour() + stamp, _, _ = _marker_stamp(tile) + seg = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + seg[_CENTRE - 1:_CENTRE + 2, _CENTRE - 1:_CENTRE + 2] = 1 + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + for i in range(len(stamp.gals)): + gal = 1.0e3 + stamp.gals[i] + gal_out, w_out, _ = _stamp_epoch_weights( + stamp, i, seed=i, blend_handling="uberseg", seg=seg, + object_number=1, + ) + npt.assert_array_equal(gal_out, gal) + assert np.all(w_out > 0.0) + + +def _develop_noisefill(gal, weight, flag, rng, bkg_rms=None): + """prepare_ngmix_weights under BLEND_HANDLING = noisefill on develop + (240b37e4), where the markers arrive as flag 2**10: the reference the + noisefill output is pinned to.""" + mask = np.copy(weight) != 0 + mask[flag != 0] = False + if bkg_rms is None: + sig_noise = sigma_mad(gal) + weight_map = mask.astype(float) / sig_noise ** 2 + else: + valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) + mask &= valid_rms + weight_map = np.zeros_like(gal, dtype=float) + weight_map[mask] = 1.0 / bkg_rms[mask] ** 2 + sig_noise = np.where(valid_rms, bkg_rms, np.median(bkg_rms[mask])) + noise_img = rng.standard_normal(gal.shape) * sig_noise + noise_img_gal = rng.standard_normal(gal.shape) * sig_noise + gal_masked = np.copy(gal) + gal_masked[~mask] = noise_img_gal[~mask] + return gal_masked, weight_map, noise_img + + +@given( + seed=st.integers(0, 2**31 - 1), + with_rms=st.booleans(), + rms_scale=st.floats(0.5, 2.0), + with_defects=st.booleans(), +) +def test_noisefill_matches_develop_on_marked_neighbours( + seed, with_rms, rms_scale, with_defects, +): + """For an epoch with a marked neighbour, noisefill returns the image, + weight and noise image develop returned, bit for bit: with no defect, + and with a flagged column and a dead pixel under the default noise fill. + + Failure mode: carrying the markers apart from the flags changes what + noisefill does to neighbour pixels, their weights, the noise level or + the RNG stream (noisefill-matches-develop). + """ + rng = np.random.default_rng(seed) + n = 31 + gal = rng.normal(0.0, 1.0, (n, n)) + gal[n // 2 - 2:n // 2 + 3, n // 2 - 2:n // 2 + 3] += 50.0 + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + neighbour = np.zeros((n, n), dtype=bool) + neighbour[n // 2 - 3:n // 2 + 4, n // 2 + 3:n // 2 + 9] = True + gal[neighbour] += 30.0 + if with_defects: + flag[:, 2] = 1 + weight[n - 3, n // 2] = 0.0 + bkg_rms = ( + rms_scale * (1.0 + 0.1 * rng.random((n, n))) if with_rms else None + ) + + new = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(seed), bkg_rms=bkg_rms, + blend_handling="noisefill", neighbour=neighbour, + ) + old = _develop_noisefill( + gal, weight, np.where(neighbour, 2**10, flag), + np.random.RandomState(seed), bkg_rms=bkg_rms, + ) + for a, b in zip(new, old): + npt.assert_array_equal(a, b) + + +def test_do_ngmix_metacal_threads_each_epochs_neighbour_mask(monkeypatch): + """Each epoch's neighbour mask reaches make_ngmix_observation. + + Failure mode: the mask is built but never used, so noisefill silently + stops filling neighbours. + """ + tile = _tile_with_neighbour() + stamp, _, _ = _marker_stamp(tile) + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + seen = [] + + class _Stop(Exception): + pass + + def fake_observation(*args, **kwargs): + seen.append(kwargs["neighbour"]) + if len(seen) == len(stamp.gals): + raise _Stop + return None + + monkeypatch.setattr( + ngmix_module, "make_ngmix_observation", fake_observation, + ) + monkeypatch.setattr(ngmix_module, "ObsList", list) + with pytest.raises(_Stop): + ngmix_module.do_ngmix_metacal( + stamp, None, 1.0, np.random.RandomState(0), + ) + for got, want in zip(seen, stamp.neighbours): + assert got is want From 2517887cd49194969034df6948ba5bde44ba0a17 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:22:58 +0200 Subject: [PATCH 09/44] fix(ngmix): keep SExtractor's neighbour markers out of the defect set prepare_postage_stamps wrote the tile VIGNET's -1e30 markers (other detections' footprints) into each epoch's flag stamp as 2**10, so defect_mask counted them: they fed the masked-fraction cut and the central veto. Every epoch shares the tile VIGNET, so a neighbour within EPOCH_CENTRAL_DEFECT_RADIUS dropped every epoch of the object. The markers are now their own per-epoch mask, stamp.neighbours (MegaCam-flipped with the epoch), threaded through do_ngmix_metacal and make_ngmix_observation to prepare_ngmix_weights. The epoch cuts count genuine defects only. Under noisefill the marked pixels get weight 0 and the defects' noise realisation, the output noisefill gave when they arrived as flags; under uberseg they are ignored and the seg-based weight handles neighbours. Defects are treated identically under both. Docstrings, @sc contracts, runner comments and the astra.yaml decisions say so. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 48 +++--- src/shapepipe/modules/ngmix_package/ngmix.py | 150 +++++++++++++------ src/shapepipe/modules/ngmix_runner.py | 15 +- tests/module/test_ngmix_uberseg.py | 3 +- 4 files changed, 143 insertions(+), 73 deletions(-) diff --git a/astra.yaml b/astra.yaml index 978fe94a7..493922a10 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1329,10 +1329,11 @@ analyses: weighted pixels within about a PSF width. DES's ngmixer fills for that reason: "it may be important for codes that take moments or use FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = - noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or - the stamp's robust noise scale otherwise; `BLEND_HANDLING = uberseg` - additionally zeroes the weight of neighbour-side pixels and leaves - their image values untouched. The committed fill + noise, defect pixels are replaced with independent noise at the + per-pixel background RMS when supplied, or the stamp's robust noise + scale otherwise. SExtractor's -1e30 neighbour markers in the tile + VIGNET are not defects; blend_handling decides their treatment. The + committed fill uses the unsymmetrized defect set: DES symmetrized its masks, but four-fold symmetrization quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in @@ -1391,10 +1392,14 @@ analyses: label: Neighbour treatment before metacal rationale: >- Covers only pixels shared with a neighbour; defect_fill does not - depend on it: defects are noise-filled under every BLEND_HANDLING. - noisefill (the default; the committed config - sets no key) leaves neighbours fully weighted and untouched. - uberseg zeroes the weight of pixels nearer a neighbour's coadd + depend on it: defects are filled the same way under every + BLEND_HANDLING, and the epoch cuts never count neighbour pixels. + noisefill (the default; the committed config sets no key) gives + weight 0 to the pixels SExtractor marks -1e30 in the tile VIGNET + (other detections' footprints; about 93% of neighbour-footprint + pixels on a SExtractor-mode sims tile) and replaces them with noise; unmarked + neighbour pixels keep their weight and light. uberseg ignores the + markers and zeroes the weight of pixels nearer a neighbour's coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) and leaves the image untouched, as official uberseg does: esheldon/meds get_uberseg returns a weight map (a @@ -1418,12 +1423,13 @@ analyses: default: noisefill options: noisefill: - label: No neighbour treatment (BLEND_HANDLING = noisefill) + label: Noise-fill SExtractor-marked neighbour pixels (BLEND_HANDLING = noisefill) description: >- - Neighbour pixels keep their full weight and image values, so the - fit sees all neighbour light, which biases shapes toward - neighbours (Jarvis et al. 2016); this masks less than even their - plain segmentation map. A candidate cause of the FLAGS=2 B-modes + Pixels marked -1e30 in the tile VIGNET get weight 0 and noise. + The fill stops at the marked footprint, so unmarked neighbour + pixels and the neighbour's wings keep their weight and light, + which biases shapes toward neighbours (Jarvis et al. 2016). A + candidate cause of the FLAGS=2 B-modes investigated in #814. insights: [mask_uberseg_neighbour_bias] uberseg: @@ -1456,9 +1462,10 @@ analyses: rationale: >- An epoch is dropped when a defect pixel lies strictly closer to the stamp centre than its fill's radius, beside the masked-fraction cut - in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. The - veto reads only the - defect mask, so it selects on nothing shear-responsive; for the same + in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. + SExtractor's -1e30 neighbour markers are not defects: all epochs + share the tile VIGNET, so a neighbour inside the radius would drop + every epoch. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same reason the radii are fixed rather than scaled by galaxy size: 10 px for noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS, @@ -1492,10 +1499,11 @@ analyses: rationale: >- An epoch whose stamp has more than EPOCH_MASKED_FRACTION_CUT (default 1/3) of its pixels in the raw, unsymmetrized defect set is - dropped from the multi-epoch fit: flagged pixels (any nonzero flag - bit, including the tile-coverage bit 2**10 set where the tile vignet - is off-image), zero-weight pixels and invalid-RMS pixels, the set - defect_fill noise-fills. An object with no surviving epoch has no + dropped from the multi-epoch fit: flagged pixels (any nonzero + exposure flag bit), zero-weight pixels and invalid-RMS pixels, the + set defect_fill fills. SExtractor's -1e30 neighbour markers in the + tile VIGNET are not counted: every epoch shares the tile VIGNET, so + a large neighbour would drop them all. An object with no surviving epoch has no shape. Before the cut, an epoch is dropped silently if its galaxy stamp is all zeros or its background-subtracted sigma_mad is not positive. DES was stricter: diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 5bd942913..1e9d10797 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -603,6 +603,11 @@ def __init__( self.psfs = [] self.weights = [] self.flags = [] + # Neighbour masks, one per epoch: the pixels SExtractor marks -1e30 in + # the tile VIGNET (other detections' footprints), MegaCam-flipped to + # the epoch. noisefill zero-weights and noise-fills them; uberseg and + # the epoch cuts do not read them (see prepare_ngmix_weights). + self.neighbours = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend # handling; empty under the default "noisefill". All epochs carry the @@ -754,10 +759,12 @@ class Ngmix(object): adaptive-moment centroid measured from the stamp pixels. See :func:`make_ngmix_observation`. blend_handling : {"noisefill", "uberseg"}, optional - Neighbour treatment. ``"noisefill"`` (default) leaves neighbour pixels - untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels - from the coadd segmentation map and requires ``seg_cat_path``. Defect - pixels are filled under both (see :func:`prepare_ngmix_weights`). + Neighbour treatment. ``"noisefill"`` (default) zero-weights and + noise-fills the pixels SExtractor marks -1e30 in the tile VIGNET + (other detections' footprints); ``"uberseg"`` ignores those markers, + zeroes the weight of neighbour-side pixels from the coadd + segmentation map and requires ``seg_cat_path``. Defect pixels are + filled under both (see :func:`prepare_ngmix_weights`). seg_cat_path : str, optional Path to the coadd-frame segmentation VIGNET catalogue (see :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. @@ -765,11 +772,11 @@ class Ngmix(object): Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. epoch_central_defect_radius : float, optional - Drop an epoch when a masked pixel lies closer than this many pixels + Drop an epoch when a defect pixel lies closer than this many pixels to the stamp centre (see :func:`has_central_defect`); the default is ``EPOCH_CENTRAL_DEFECT_RADIUS``. epoch_masked_fraction_cut : float, optional - Drop an epoch when more than this fraction of its stamp is masked + Drop an epoch when more than this fraction of its stamp is defects (see :func:`prepare_postage_stamps`); the default is ``EPOCH_MASKED_FRACTION_CUT``. defect_fill : {"noise", "interpolate"}, optional @@ -1511,10 +1518,19 @@ def prepare_postage_stamps( An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS pixels, the set that :func:`prepare_ngmix_weights` zero-weights and - fills. Counting flags alone would keep epochs whose filled area exceeds - the cut. The zero-weight orbit of interpolated pixels keeps its light and - is not counted. The default is 1/3; 10%, the DES Y3 and Y6 value, is - the alternative to test. + fills under every ``blend_handling``. Counting flags alone would keep + epochs whose filled area exceeds the cut. The zero-weight orbit of + interpolated pixels keeps its light and is not counted. The default is + 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects + SExtractor writes -1e30 into the tile VIGNET on the footprints of other + detections (and beyond the tile's edge). These markers form the epoch's neighbour mask + (``stamp.neighbours``, MegaCam-flipped like the epoch), kept apart from + its flag stamp, so neither the masked-fraction cut nor the central + veto counts them. Every epoch shares the tile VIGNET: counting a + neighbour within the veto radius as a defect would drop every epoch of + the object. Parameters ---------- @@ -1618,8 +1634,12 @@ def prepare_postage_stamps( tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) flag_vign = flag_obj[expccd_name]['VIGNET'] - if tile_vign is not None: - flag_vign[np.where(tile_vign == -1e30)] = 2**10 + # Neighbour markers (neighbour-markers-are-not-defects). + neighbour = ( + tile_vign == -1e30 + if tile_vign is not None + else np.zeros(np.shape(gal_vign), dtype=bool) + ) weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] @@ -1673,6 +1693,7 @@ def prepare_postage_stamps( stamp.psfs.append(psf_obj[expccd_name]['VIGNET']) stamp.weights.append(weight_vign_scaled) stamp.flags.append(flag_vign) + stamp.neighbours.append(neighbour) stamp.bkg_rms.append(bkg_rms_vign_scaled) if tile_seg is not None: stamp.segs.append(tile_seg) @@ -1981,11 +2002,13 @@ def defect_mask(weight, flag, bkg_rms=None): """Defect pixels of one epoch stamp. @sc [decision:shape_measurement.defect_fill,label:physics] defect-set-unsymmetrized - A defect is a pixel with zero exposure weight, a nonzero flag, or (when a - background RMS map is given) a non-finite or non-positive RMS. This one - set is zero-weighted and noise-filled by :func:`prepare_ngmix_weights` - and counted by the epoch cuts in :func:`prepare_postage_stamps`. It is - not ORed with its rotations. For defects the central-defect veto keeps + A defect is a pixel with zero exposure weight, a nonzero exposure flag, + or (when a background RMS map is given) a non-finite or non-positive RMS. + This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` + under every ``blend_handling`` and counted by the epoch cuts in + :func:`prepare_postage_stamps`. SExtractor's neighbour markers are not + in it (neighbour-markers-are-not-defects). It is not ORed with its + rotations. For defects the central-defect veto keeps (:func:`has_central_defect`), the unsymmetrized fill leaves |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" and 0.5" through a 0.7" PSF, round or elliptical. Symmetrizing would @@ -2001,7 +2024,7 @@ def defect_mask(weight, flag, bkg_rms=None): weight : numpy.ndarray Exposure weight stamp. flag : numpy.ndarray - Flag stamp. + Exposure flag stamp. bkg_rms : numpy.ndarray, optional Background RMS stamp. @@ -2130,7 +2153,8 @@ def fill_defects(image, defect, noise): image : numpy.ndarray Stamp image. defect : numpy.ndarray of bool - Pixels to replace (:func:`defect_mask`). + Pixels to replace: the defects (:func:`defect_mask`), and under + noisefill the marked neighbour pixels. noise : numpy.ndarray Noise realisation on the stamp grid. @@ -2145,7 +2169,7 @@ def fill_defects(image, defect, noise): def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, defect_fill="noise", + dilate_neighbour=0, defect_fill="noise", neighbour=None, ): """Build one epoch's image, weight map and noise image for ngmix. @@ -2154,17 +2178,29 @@ def prepare_ngmix_weights( by an independent noise realisation at their background RMS (:func:`fill_defects`), or under ``defect_fill="interpolate"``, for short defect runs, by an interpolant of the clean pixels around them. - ``blend_handling`` decides only the neighbour side. + ``blend_handling`` decides only the neighbour pixels. - @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-neighbours-raw + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-whatever-the-blend-handling Every pixel of ``defect_mask`` is zero-weighted and filled whatever - ``blend_handling`` is, and the filled set equals the defect set. - ``blend_handling`` acts on the neighbour side only: uberseg zeroes - the weight of pixels nearer a neighbour's footprint and leaves their - image values raw, because that light is real sky that metacal shears - along with the target; filling it with noise would cut the target along - an unsheared edge. Neighbour pixels are never filled. The noise image - covers the whole stamp and is independent of both. + ``blend_handling`` is, by the same operator. ``blend_handling`` acts on + the neighbour pixels only, and the noise image covers the whole stamp. + + @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers + Under ``"noisefill"`` the pixels of ``neighbour``, SExtractor's -1e30 + neighbour markers, get weight 0 and are replaced by the same noise + realisation as the noise-filled defects, so no marked neighbour light + reaches metacal. Under the default noise fill, the image, weight map and + noise image are those the marked pixels would get as flagged defects; + only the epoch cuts treat them differently, by not counting them + (neighbour-markers-are-not-defects). Unmarked neighbour light stays + raw and weighted. + + @sc [decision:shape_measurement.blend_handling] uberseg-ignores-markers + Under ``"uberseg"`` the markers are ignored: :func:`uberseg_weight` + zeroes the weight of pixels nearer a neighbour's segmentation footprint + than the target's and leaves their image values raw, because that light + is real sky that metacal shears along with the target; filling it with + noise would cut the target along an unsheared edge. @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defect-weight-orbit Under ``defect_fill="interpolate"`` the short defect runs @@ -2189,7 +2225,7 @@ def prepare_ngmix_weights( weight : numpy.ndarray Exposure weight stamp; zero marks a defect. flag : numpy.ndarray - Flag stamp; nonzero marks a defect. + Exposure flag stamp; nonzero marks a defect. rng : numpy.random.RandomState Random state for the noise realisations (seeded per object; see :func:`position_seed`). @@ -2199,11 +2235,11 @@ def prepare_ngmix_weights( non-positive values mark defects. Otherwise every clean pixel gets ``1 / sigma_mad(gal)**2``. blend_handling : {"noisefill", "uberseg"}, optional - Neighbour treatment. ``"noisefill"`` (default) leaves neighbour - pixels weighted and untouched. ``"uberseg"`` zeroes the weight of - every pixel closer to a neighbour's segmentation footprint than to the - central object's and keeps its raw image value (see - :func:`uberseg_weight`). + Neighbour treatment. ``"noisefill"`` (default) zero-weights and + noise-fills the ``neighbour`` pixels. ``"uberseg"`` ignores + ``neighbour``, zeroes the weight of every pixel closer to a + neighbour's segmentation footprint than to the central object's and + keeps its raw image value (see :func:`uberseg_weight`). seg : numpy.ndarray, optional Segmentation map on the stamp grid (object NUMBERs). Required for ``blend_handling="uberseg"``; ignored otherwise. @@ -2216,11 +2252,15 @@ def prepare_ngmix_weights( defect_fill : {"noise", "interpolate"}, optional ``"noise"`` (default) noise-fills every defect; ``"interpolate"`` interpolates the short defect runs and noise-fills the rest. + neighbour : numpy.ndarray of bool, optional + The epoch's neighbour mask (``Postage_stamp.neighbours``); read only + under ``blend_handling="noisefill"``. ``None`` marks no pixel. Returns ------- numpy.ndarray - Galaxy image with defect pixels filled. + Galaxy image with defect pixels, and under noisefill marked + neighbour pixels, filled. numpy.ndarray Inverse-variance weight map for ngmix. numpy.ndarray @@ -2253,7 +2293,13 @@ def prepare_ngmix_weights( ) defect = defect_mask(weight, flag, bkg_rms) - clean = ~defect + # Marked neighbour pixels that noisefill removes (noisefill-fills-markers). + removed_neighbour = ( + np.asarray(neighbour, dtype=bool) + if blend_handling == "noisefill" and neighbour is not None + else np.zeros_like(defect) + ) + clean = ~(defect | removed_neighbour) interpolated = ( interpolable_defects(defect) if defect_fill == "interpolate" @@ -2262,7 +2308,7 @@ def prepare_ngmix_weights( # The quarter-turn orbit of the interpolated pixels carries no weight # (interpolated-defect-weight-orbit). weighted = ( - ~(defect | fourfold(interpolated)) if interpolated.any() else clean + clean & ~fourfold(interpolated) if interpolated.any() else clean ) if bkg_rms is None: @@ -2299,12 +2345,17 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise - gal_filled = fill_defects(gal, defect, noise_img_gal) + gal_filled = fill_defects(gal, ~clean, noise_img_gal) if interpolated.any(): # One operator for the image and the noise image; a pixel whose - # support is degenerate keeps its noise fill. + # support is degenerate, or that noisefill removes as a neighbour, + # keeps its noise fill. filled = interpolate_defects([gal, noise_img], defect, interpolated) - done = interpolated & np.all(np.isfinite(filled), axis=0) + done = ( + interpolated + & ~removed_neighbour + & np.all(np.isfinite(filled), axis=0) + ) gal_filled[done] = filled[0][done] noise_img = np.where(done, filled[1], noise_img) @@ -2320,7 +2371,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, defect_fill="noise", + dilate_neighbour=0, defect_fill="noise", neighbour=None, ): """Build an ngmix Observation for a single galaxy epoch. @@ -2364,7 +2415,8 @@ def make_ngmix_observation( ``"hsm"``). blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to :func:`prepare_ngmix_weights`; - the default ``"noisefill"`` leaves neighbour pixels untouched. + the default ``"noisefill"`` zero-weights and noise-fills the + ``neighbour`` pixels. seg : numpy.ndarray, optional Segmentation map on the stamp grid. Required for ``blend_handling="uberseg"`` (ignored otherwise). @@ -2377,6 +2429,8 @@ def make_ngmix_observation( defect_fill : {"noise", "interpolate"}, optional Defect fill passed through to :func:`prepare_ngmix_weights`; the default is ``"noise"``. + neighbour : numpy.ndarray of bool, optional + Neighbour mask passed through to :func:`prepare_ngmix_weights`. Returns ------- @@ -2406,6 +2460,7 @@ def make_ngmix_observation( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, + neighbour=neighbour, ) if centroid_source == "hsm": @@ -2661,9 +2716,9 @@ def do_ngmix_metacal( stamp pixels — see that function. blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to - :func:`make_ngmix_observation`; the default ``"noisefill"`` leaves - neighbour pixels untouched. ``"uberseg"`` consumes ``stamp.segs`` and - ``object_number``. + :func:`make_ngmix_observation`; the default ``"noisefill"`` + zero-weights and noise-fills the pixels of ``stamp.neighbours``. + ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional Central object's segmentation label — its SExtractor ``NUMBER`` (``obj_id``), authoritative because seg labels are the NUMBERs of the @@ -2720,6 +2775,9 @@ def do_ngmix_metacal( object_number=object_number, dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, + neighbour=( + stamp.neighbours[n_e] if n_e < len(stamp.neighbours) else None + ), ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 166e8cae5..dc27b8afc 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -144,10 +144,11 @@ def ngmix_runner( else: centroid_source = "wcs" - # Neighbour treatment: "noisefill" (default) leaves neighbour pixels - # weighted and untouched; "uberseg" zeroes the weight of every pixel - # closer to a neighbour than to the central object, from the segmentation - # map, and leaves its image raw. Defect pixels (flagged, zero-weight or + # Neighbour treatment: "noisefill" (default) zero-weights and noise-fills + # the pixels SExtractor marks -1e30 in the tile VIGNET (other detections' + # footprints); "uberseg" ignores those markers, zeroes the weight of every + # pixel closer to a neighbour than to the central object, from the + # segmentation map, and leaves its image raw. Defect pixels (flagged, zero-weight or # invalid-RMS) are zero-weighted and filled under both (DEFECT_FILL # below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): @@ -164,7 +165,8 @@ def ngmix_runner( dilate_neighbour = 1 # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a - # masked pixel lies closer than this to the stamp centre; 0 disables. + # defect pixel (flagged, zero-weight or invalid-RMS; not a neighbour + # marker) lies closer than this to the stamp centre; 0 disables. if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): epoch_central_defect_radius = config.getfloat( module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" @@ -173,7 +175,8 @@ def ngmix_runner( epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this - # fraction of its stamp is masked (flagged, zero-weight or invalid-RMS). + # fraction of its stamp is defects (flagged, zero-weight or invalid-RMS; + # neighbour markers are not counted). if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): epoch_masked_fraction_cut = config.getfloat( module_config_sec, "EPOCH_MASKED_FRACTION_CUT" diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 92616fde0..5c20789f6 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -263,7 +263,8 @@ def test_uberseg_fills_defects_and_leaves_neighbour_pixels_raw(): Failure modes: the defect fill is skipped under uberseg, so raw bad pixels reach metacal; or the neighbour side is noise-filled - (defects-filled-neighbours-raw). + (defects-filled-whatever-the-blend-handling, + uberseg-ignores-markers). """ npix = 41 gal, flag, weight = _gal_flag_weight(npix=npix) From 7faa9e78dcd7118a1308563a7c4c322741b08fc5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:45:43 +0200 Subject: [PATCH 10/44] fix(ngmix): off-tile pixels are defects, not neighbours SExtractor writes -1e30 into the tile VIGNET beyond the tile's edge as well as on other detections' footprints. Beyond the edge the epoch holds the object's own light, cut off, so treating those pixels as neighbours exempted edge objects from both epoch cuts and measured them with a noise-filled band near their centre. split_tile_markers classifies the markers geometrically on the flipped tile VIGNET: the union of stamp rows and columns that are entirely -1e30 (the off-image part of a rectangle clip) is off-tile, the rest is the neighbour mask. Off-tile pixels are flagged OFF_TILE_FLAG (2**10) in the epoch's flag stamp, so under both blend handlings they are zero-weighted, filled, and counted by the masked-fraction cut and the central veto. Docstrings, @sc contracts, runner comments and astra.yaml say so. Tests: an object 3 px from the tile edge loses every epoch; a band 12 px away passes the default cuts and is vetoed at radius 13; a corner's L-shaped off-tile region is flagged exactly and a border-touching neighbour stays a neighbour; off-tile pixels are filled under both blend handlings. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 33 +++-- src/shapepipe/modules/ngmix_package/ngmix.py | 77 +++++++++--- src/shapepipe/modules/ngmix_runner.py | 14 +-- tests/module/test_ngmix_defect_fill.py | 121 ++++++++++++++++++- 4 files changed, 206 insertions(+), 39 deletions(-) diff --git a/astra.yaml b/astra.yaml index 493922a10..fde5b8a70 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1321,8 +1321,8 @@ analyses: defect_fill: label: Image content of defect pixels before metacal rationale: >- - A defect pixel (nonzero instrument flag, zero exposure weight or - invalid background RMS) gets weight 0 in prepare_ngmix_weights. Its + A defect pixel (nonzero instrument flag, zero exposure weight, + invalid background RMS, or off-tile) gets weight 0 in prepare_ngmix_weights. Its image value still matters: ngmix's metacal deconvolves, shears and reconvolves an InterpolatedImage of the whole image and copies the weights through, so a zero-weight pixel's content spreads into the @@ -1331,9 +1331,13 @@ analyses: FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise - scale otherwise. SExtractor's -1e30 neighbour markers in the tile - VIGNET are not defects; blend_handling decides their treatment. The - committed fill + scale otherwise. SExtractor writes -1e30 into the tile VIGNET on + other detections' footprints and beyond the tile's edge. The stamp + rows and columns that are entirely -1e30 (the off-image part of a + rectangle clip) are off-tile: the epoch holds the object's own light + there, so they are defects, flagged 2**10. The other markers are + neighbour pixels, not defects; blend_handling decides their + treatment. The committed fill uses the unsymmetrized defect set: DES symmetrized its masks, but four-fold symmetrization quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in @@ -1396,8 +1400,10 @@ analyses: BLEND_HANDLING, and the epoch cuts never count neighbour pixels. noisefill (the default; the committed config sets no key) gives weight 0 to the pixels SExtractor marks -1e30 in the tile VIGNET - (other detections' footprints; about 93% of neighbour-footprint - pixels on a SExtractor-mode sims tile) and replaces them with noise; unmarked + on other detections' footprints (about 93% of neighbour-footprint + pixels on a SExtractor-mode sims tile), excluding the off-tile + rows and columns, which are defects, and replaces them with noise; + unmarked neighbour pixels keep their weight and light. uberseg ignores the markers and zeroes the weight of pixels nearer a neighbour's coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) @@ -1465,7 +1471,8 @@ analyses: in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. SExtractor's -1e30 neighbour markers are not defects: all epochs share the tile VIGNET, so a neighbour inside the radius would drop - every epoch. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same + every epoch. Off-tile pixels are defects, so an object near the + tile edge is vetoed. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same reason the radii are fixed rather than scaled by galaxy size: 10 px for noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS, @@ -1500,10 +1507,12 @@ analyses: An epoch whose stamp has more than EPOCH_MASKED_FRACTION_CUT (default 1/3) of its pixels in the raw, unsymmetrized defect set is dropped from the multi-epoch fit: flagged pixels (any nonzero - exposure flag bit), zero-weight pixels and invalid-RMS pixels, the - set defect_fill fills. SExtractor's -1e30 neighbour markers in the - tile VIGNET are not counted: every epoch shares the tile VIGNET, so - a large neighbour would drop them all. An object with no surviving epoch has no + exposure flag bit), zero-weight pixels, invalid-RMS pixels and + off-tile pixels (whole -1e30 rows and columns of the tile VIGNET, + flag 2**10), the set defect_fill fills. On a 51-px stamp an object + within about 8.5 px of the tile edge fails the 1/3 cut. The other + -1e30 markers, neighbour footprints, are not counted: every epoch + shares the tile VIGNET, so a large neighbour would drop them all. An object with no surviving epoch has no shape. Before the cut, an epoch is dropped silently if its galaxy stamp is all zeros or its background-subtracted sigma_mad is not positive. DES was stricter: diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 1e9d10797..9cd196743 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -30,6 +30,9 @@ ) from shapepipe.pipeline import file_io +# Flag bit set on an epoch's off-tile pixels (see :func:`split_tile_markers`). +OFF_TILE_FLAG = 2**10 + # Neighbour treatments selectable with the BLEND_HANDLING option. BLEND_HANDLINGS = ("noisefill", "uberseg") @@ -604,8 +607,9 @@ def __init__( self.weights = [] self.flags = [] # Neighbour masks, one per epoch: the pixels SExtractor marks -1e30 in - # the tile VIGNET (other detections' footprints), MegaCam-flipped to - # the epoch. noisefill zero-weights and noise-fills them; uberseg and + # the tile VIGNET on other detections' footprints (off-tile markers + # are flagged as defects instead; see split_tile_markers), + # MegaCam-flipped to the epoch. noisefill zero-weights and noise-fills them; uberseg and # the epoch cuts do not read them (see prepare_ngmix_weights). self.neighbours = [] self.bkg_rms = [] @@ -761,7 +765,7 @@ class Ngmix(object): blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment. ``"noisefill"`` (default) zero-weights and noise-fills the pixels SExtractor marks -1e30 in the tile VIGNET - (other detections' footprints); ``"uberseg"`` ignores those markers, + on other detections' footprints; ``"uberseg"`` ignores those markers, zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect pixels are filled under both (see :func:`prepare_ngmix_weights`). @@ -1525,12 +1529,20 @@ def prepare_postage_stamps( @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects SExtractor writes -1e30 into the tile VIGNET on the footprints of other - detections (and beyond the tile's edge). These markers form the epoch's neighbour mask - (``stamp.neighbours``, MegaCam-flipped like the epoch), kept apart from - its flag stamp, so neither the masked-fraction cut nor the central - veto counts them. Every epoch shares the tile VIGNET: counting a - neighbour within the veto radius as a defect would drop every epoch of - the object. + detections and beyond the tile's edge (:func:`split_tile_markers`). The + footprint markers form the epoch's neighbour mask (``stamp.neighbours``, + MegaCam-flipped like the epoch), kept apart from its flag stamp, so + neither the masked-fraction cut nor the central veto counts them. Every + epoch shares the tile VIGNET: counting a neighbour within the veto + radius as a defect would drop every epoch of the object. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.defect_fill] off-tile-pixels-are-defects + Beyond the tile's edge the epoch holds real data, the object's own light + cut off by the tile. Those pixels are flagged ``OFF_TILE_FLAG`` (2**10) + in the epoch's flag stamp and so join its defect set under every + ``blend_handling``: zero weight, the defect fill, and both epoch cuts. + On a 51-px stamp an object within about 8.5 px of the tile edge fails + the 1/3 cut. Parameters ---------- @@ -1634,12 +1646,10 @@ def prepare_postage_stamps( tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) flag_vign = flag_obj[expccd_name]['VIGNET'] - # Neighbour markers (neighbour-markers-are-not-defects). - neighbour = ( - tile_vign == -1e30 - if tile_vign is not None - else np.zeros(np.shape(gal_vign), dtype=bool) - ) + # Off-tile pixels are defects (off-tile-pixels-are-defects); the + # other -1e30 markers are neighbours (neighbour-markers-are-not-defects). + neighbour, off_tile = split_tile_markers(tile_vign, np.shape(gal_vign)) + flag_vign[off_tile] = OFF_TILE_FLAG weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] @@ -1714,6 +1724,38 @@ def prepare_postage_stamps( return stamp +def split_tile_markers(tile_vign, shape): + """Split the tile VIGNET's -1e30 markers into neighbour and off-tile. + + @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill] off-tile-is-whole-marked-rows-and-columns + SExtractor writes -1e30 on the footprints of other detections and on + stamp pixels beyond the tile's edge. A stamp clipped by the tile's + rectangle loses whole rows and whole columns, so the off-tile pixels are + the union of the stamp rows and columns that are entirely -1e30. The + remaining markers are neighbour pixels; a footprint touching the stamp + border stays a neighbour unless it fills a whole row or column. + + Parameters + ---------- + tile_vign : numpy.ndarray or None + Tile VIGNET stamp, oriented like the epoch; ``None`` marks nothing. + shape : tuple of int + Stamp shape, used when ``tile_vign`` is ``None``. + + Returns + ------- + numpy.ndarray of bool + Neighbour pixels. + numpy.ndarray of bool + Off-tile pixels. + """ + if tile_vign is None: + return np.zeros(shape, dtype=bool), np.zeros(shape, dtype=bool) + marker = tile_vign == -1e30 + off_tile = marker.all(axis=1)[:, None] | marker.all(axis=0)[None, :] + return marker & ~off_tile, off_tile + + def background_subtract(gal,bkg): """background subtraction. @@ -2007,7 +2049,8 @@ def defect_mask(weight, flag, bkg_rms=None): This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` under every ``blend_handling`` and counted by the epoch cuts in :func:`prepare_postage_stamps`. SExtractor's neighbour markers are not - in it (neighbour-markers-are-not-defects). It is not ORed with its + in it (neighbour-markers-are-not-defects); off-tile pixels are, as flag + ``OFF_TILE_FLAG`` (off-tile-pixels-are-defects). It is not ORed with its rotations. For defects the central-defect veto keeps (:func:`has_central_defect`), the unsymmetrized fill leaves |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" @@ -2196,7 +2239,7 @@ def prepare_ngmix_weights( raw and weighted. @sc [decision:shape_measurement.blend_handling] uberseg-ignores-markers - Under ``"uberseg"`` the markers are ignored: :func:`uberseg_weight` + Under ``"uberseg"`` the neighbour markers are ignored: :func:`uberseg_weight` zeroes the weight of pixels nearer a neighbour's segmentation footprint than the target's and leaves their image values raw, because that light is real sky that metacal shears along with the target; filling it with diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index dc27b8afc..e217995c2 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -148,9 +148,9 @@ def ngmix_runner( # the pixels SExtractor marks -1e30 in the tile VIGNET (other detections' # footprints); "uberseg" ignores those markers, zeroes the weight of every # pixel closer to a neighbour than to the central object, from the - # segmentation map, and leaves its image raw. Defect pixels (flagged, zero-weight or - # invalid-RMS) are zero-weighted and filled under both (DEFECT_FILL - # below); see prepare_ngmix_weights. + # segmentation map, and leaves its image raw. Defect pixels (flagged, + # zero-weight, invalid-RMS or off-tile) are zero-weighted and filled + # under both (DEFECT_FILL below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: @@ -165,8 +165,8 @@ def ngmix_runner( dilate_neighbour = 1 # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a - # defect pixel (flagged, zero-weight or invalid-RMS; not a neighbour - # marker) lies closer than this to the stamp centre; 0 disables. + # defect pixel (flagged, zero-weight, invalid-RMS or off-tile; not a + # neighbour marker) lies closer than this to the stamp centre; 0 disables. if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): epoch_central_defect_radius = config.getfloat( module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" @@ -175,8 +175,8 @@ def ngmix_runner( epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this - # fraction of its stamp is defects (flagged, zero-weight or invalid-RMS; - # neighbour markers are not counted). + # fraction of its stamp is defects (flagged, zero-weight, invalid-RMS or + # off-tile; neighbour markers are not counted). if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): epoch_masked_fraction_cut = config.getfloat( module_config_sec, "EPOCH_MASKED_FRACTION_CUT" diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index b9e3096c5..9c15d1807 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -786,7 +786,7 @@ def _expected_neighbours(tile, name): def test_neighbour_markers_near_the_centre_keep_every_epoch(): - """A neighbour footprint 3 px from the centre, and one covering 45% of + """A neighbour footprint 3 px from the centre, and one covering 41% of the stamp, drop no epoch: the masked-fraction cut and the central veto count no marker. @@ -796,8 +796,10 @@ def test_neighbour_markers_near_the_centre_keep_every_epoch(): (neighbour-markers-are-not-defects). """ small = _tile_with_neighbour() - large = _tile_with_neighbour(rows=(0, N_STAMP)) - large[:, _CENTRE + 3:] = _MARKER + # Large, but short of the stamp border: no row or column is entirely + # marked, so it is a neighbour, not off-tile. + large = _tile_with_neighbour() + large[1:-1, _CENTRE + 3:-1] = _MARKER assert (large == _MARKER).mean() > 1 / 3 for tile in (small, large): stamp, _, _ = _marker_stamp(tile) @@ -939,6 +941,11 @@ def test_noisefill_matches_develop_on_marked_neighbours( """For an epoch with a marked neighbour, noisefill returns the image, weight and noise image develop returned, bit for bit: with no defect, and with a flagged column and a dead pixel under the default noise fill. + The stamps carry no off-tile pixels, and the equivalence is claimed for + such stamps only. Off-tile pixels reach this function as flag 2**10 + defects (off-tile-pixels-are-defects), as every marker did on develop; + which epochs survive the cuts differs from develop wherever there are + neighbour markers. Failure mode: carrying the markers apart from the flags changes what noisefill does to neighbour pixels, their weights, the noise level or @@ -1002,3 +1009,111 @@ def fake_observation(*args, **kwargs): ) for got, want in zip(seen, stamp.neighbours): assert got is want + + +# --- Off-tile pixels are defects ------------------------------------------- +# +# SExtractor also writes -1e30 beyond the tile's edge, where the epoch holds +# the object's own light, cut off. Those pixels are the stamp rows and +# columns that are entirely -1e30 (the off-image part of a rectangle clip); +# they join the epoch's defect set as flag 2**10. The other markers are the +# neighbour mask. + +_OFF_TILE = 2**10 + + +def _off_tile_expected(tile, name): + flipped = Ngmix.MegaCamFlip(tile, int(name.split("-")[1])) == _MARKER + return flipped.all(axis=1)[:, None] | flipped.all(axis=0)[None, :] + + +def test_object_three_px_from_the_tile_edge_is_dropped(): + """With the tile edge 3 px from the object, every epoch is dropped: the + off-tile band counts toward the epoch cuts. + + Failure mode: off-tile pixels are treated as neighbour markers, so an + edge object is measured with a noise-filled band through its own light + (off-tile-pixels-are-defects). + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:, :_CENTRE - 2] = _MARKER + stamp, _, _ = _marker_stamp(tile) + assert len(stamp.gals) == 0 + assert stamp.epoch_cuts["considered"] == len(_MARKER_EPOCH_NAMES) + assert ( + stamp.epoch_cuts["masked_fraction"] + stamp.epoch_cuts["central_veto"] + == len(_MARKER_EPOCH_NAMES) + ) + + +def test_the_central_veto_sees_off_tile_pixels(): + """An off-tile band 12 px from the object (27% of the stamp) passes the + default cuts and is vetoed at radius 13. + + Failure mode: the central veto does not read the off-tile set. + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:, :_CENTRE - 11] = _MARKER + kept, _, _ = _marker_stamp(tile) + assert len(kept.gals) == len(_MARKER_EPOCH_NAMES) + vetoed, _, _ = _marker_stamp(tile, epoch_central_defect_radius=13) + assert len(vetoed.gals) == 0 + assert vetoed.epoch_cuts["central_veto"] == len(_MARKER_EPOCH_NAMES) + + +def test_corner_off_tile_region_and_border_neighbour_are_classified(): + """At a tile corner, the L-shaped off-tile region is flagged 2**10 + exactly, and a neighbour footprint touching the stamp border without + filling a row or column stays in the neighbour mask. + + Failure modes: off-tile pixels are classified by something other than + whole marked rows and columns (the L is missed or a border-touching + neighbour is swallowed); the classification ignores the MegaCam flip. + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:5, :] = _MARKER + tile[:, -5:] = _MARKER + tile[40:, :6] = _MARKER # neighbour on the bottom-left border + stamp, epochs, _ = _marker_stamp(tile) + names = {id(v[0]): name for name, v in epochs.items()} + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + for flag, neighbour in zip(stamp.flags, stamp.neighbours): + name = names[id(flag)] + off_tile = _off_tile_expected(tile, name) + assert off_tile.sum() == 5 * N_STAMP * 2 - 25 + npt.assert_array_equal(flag == _OFF_TILE, off_tile) + npt.assert_array_equal( + neighbour, _expected_neighbours(tile, name) & ~off_tile + ) + assert neighbour.sum() == 11 * 6 + + +@pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) +def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): + """Off-tile pixels are zero-weighted and noise-filled under either + BLEND_HANDLING; under uberseg the neighbour markers stay raw. + + Failure mode: under uberseg the off-tile band keeps its weight, or the + fill differs between blend handlings. + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:5, :] = _MARKER + tile[20:24, 35:40] = _MARKER + stamp, _, _ = _marker_stamp(tile) + seg = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + seg[_CENTRE - 1:_CENTRE + 2, _CENTRE - 1:_CENTRE + 2] = 1 + kwargs = ( + dict(seg=seg, object_number=1) if blend_handling == "uberseg" else {} + ) + for i in range(len(stamp.gals)): + gal = 1.0e3 + stamp.gals[i] + gal_out, w_out, _ = _stamp_epoch_weights( + stamp, i, seed=i, blend_handling=blend_handling, **kwargs, + ) + off_tile = stamp.flags[i] == _OFF_TILE + assert off_tile.sum() == 5 * N_STAMP + removed = off_tile | ( + stamp.neighbours[i] if blend_handling == "noisefill" else False + ) + npt.assert_array_equal(gal_out != gal, removed) + npt.assert_array_equal(w_out == 0.0, removed) From f0654053f5e300d9ea6293afb2b5022c0895b144 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:47:26 +0200 Subject: [PATCH 11/44] fix(ngmix): interpolate defects from the pixels the image keeps Under DEFECT_FILL = interpolate and BLEND_HANDLING = noisefill, the Clough-Tocher support included marked neighbour pixels that noisefill replaces with noise, so a defect beside a bright neighbour was filled with light the image no longer contains. The support now excludes ~clean (the defects and the removed neighbour pixels); the NaN fallback to the noise fill covers degenerate supports. Under uberseg the neighbour light stays in the image and the support is unchanged. interpolate_defects names its second argument `excluded`. Test: a flagged pixel next to a 1e4 marked neighbour interpolates to the kept-pixel interpolant (1.4, against 3102 with the neighbour in the support) under noisefill, and from the raw neighbour light under uberseg. Co-Authored-By: Claude Opus 5.5 --- .../ngmix_package/defect_interpolation.py | 24 ++++----- src/shapepipe/modules/ngmix_package/ngmix.py | 18 +++++-- tests/module/test_ngmix_defect_fill.py | 52 +++++++++++++++++++ 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/defect_interpolation.py b/src/shapepipe/modules/ngmix_package/defect_interpolation.py index 713e326f7..ae0d53c41 100644 --- a/src/shapepipe/modules/ngmix_package/defect_interpolation.py +++ b/src/shapepipe/modules/ngmix_package/defect_interpolation.py @@ -94,14 +94,14 @@ def fourfold(mask): return mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) -def _interpolate_once(planes, defect, target): +def _interpolate_once(planes, excluded, target): """One Clough-Tocher interpolant of every plane at ``target``; NaN elsewhere and where the support cannot reach.""" out = np.full(planes.shape, np.nan) support = binary_dilation( target, structure=np.ones((3, 3), dtype=bool), iterations=SUPPORT_RADIUS, - ) & ~defect + ) & ~excluded points = np.argwhere(support).astype(float) if len(points) < 3: return out @@ -116,14 +116,14 @@ def _interpolate_once(planes, defect, target): return out -def interpolate_defects(planes, defect, target): +def interpolate_defects(planes, excluded, target): """Replace the ``target`` pixels of every plane by a Clough-Tocher - interpolant of the clean pixels around them. + interpolant of the kept pixels around them. @sc [decision:shape_measurement.defect_fill] shared-rotation-averaged-interpolant - The support is the clean pixels within ``SUPPORT_RADIUS`` (4 px) of the - target; no defect pixel enters it, so defect values are never read. For - each quarter turn of the stamp, one Delaunay triangulation of the support + The support is the pixels within ``SUPPORT_RADIUS`` (4 px) of the target + outside ``excluded``; no excluded pixel enters it, so their values are + never read. For each quarter turn of the stamp, one Delaunay triangulation of the support serves every plane, so the science image and the metacal noise image see the same linear operator and fixnoise mirrors the science image's interpolated noise. A regular grid's triangulation has degenerate @@ -136,9 +136,9 @@ def interpolate_defects(planes, defect, target): ---------- planes : array_like Stamp planes, shape ``(n, ny, nx)``. - defect : numpy.ndarray of bool - Every defect pixel, shape ``(ny, nx)``; none of them supports the - interpolant. + excluded : numpy.ndarray of bool + Pixels that never support the interpolant, shape ``(ny, nx)``: every + defect, and any pixel whose light the image does not keep. target : numpy.ndarray of bool Defect pixels to interpolate (:func:`interpolable_defects`). @@ -149,7 +149,7 @@ def interpolate_defects(planes, defect, target): target pixel whose support is degenerate in some orientation. """ planes = np.asarray(planes, dtype=float) - defect = np.asarray(defect, dtype=bool) + excluded = np.asarray(excluded, dtype=bool) target = np.asarray(target, dtype=bool) out = planes.copy() if not target.any(): @@ -157,7 +157,7 @@ def interpolate_defects(planes, defect, target): turns = [ np.rot90( _interpolate_once( - np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), + np.rot90(planes, k, axes=(1, 2)), np.rot90(excluded, k), np.rot90(target, k), ), -k, axes=(1, 2), diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 9cd196743..bf9bf0b12 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -2228,6 +2228,15 @@ def prepare_ngmix_weights( ``blend_handling`` is, by the same operator. ``blend_handling`` acts on the neighbour pixels only, and the noise image covers the whole stamp. + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] interpolation-support-is-the-kept-image + Under ``defect_fill="interpolate"`` the interpolant is supported on the + pixels whose light the image keeps: never a defect, and under + ``"noisefill"`` never a marked neighbour pixel, whose light is replaced + by noise; a Clough-Tocher interpolant next to a bright neighbour would + otherwise build the defect's value from light the image no longer + contains. Under ``"uberseg"`` the neighbour light stays in the image and + supports the interpolant. + @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers Under ``"noisefill"`` the pixels of ``neighbour``, SExtractor's -1e30 neighbour markers, get weight 0 and are replaced by the same noise @@ -2390,10 +2399,11 @@ def prepare_ngmix_weights( noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_filled = fill_defects(gal, ~clean, noise_img_gal) if interpolated.any(): - # One operator for the image and the noise image; a pixel whose - # support is degenerate, or that noisefill removes as a neighbour, - # keeps its noise fill. - filled = interpolate_defects([gal, noise_img], defect, interpolated) + # One operator for the image and the noise image, supported on the + # pixels the image keeps (interpolation-support-is-the-kept-image); a + # pixel whose support is degenerate, or that noisefill removes as a + # neighbour, keeps its noise fill. + filled = interpolate_defects([gal, noise_img], ~clean, interpolated) done = ( interpolated & ~removed_neighbour diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 9c15d1807..67aff0d20 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -1117,3 +1117,55 @@ def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): ) npt.assert_array_equal(gal_out != gal, removed) npt.assert_array_equal(w_out == 0.0, removed) + + +# --- DEFECT_FILL = interpolate beside a removed neighbour ------------------- + +def _defect_beside_neighbour(): + """A single flagged pixel 6 px right of the centre, with a bright marked + neighbour footprint starting on the next column.""" + n = 31 + c = n // 2 + gal = np.random.default_rng(3).normal(0.0, 1.0, (n, n)) + flag = np.zeros((n, n), dtype=np.int32) + flag[c, c + 6] = 1 + neighbour = np.zeros((n, n), dtype=bool) + neighbour[c - 2:c + 3, c + 7:c + 10] = True + gal[neighbour] += 1.0e4 + seg = np.zeros((n, n), dtype=np.int32) + seg[c - 1:c + 2, c - 1:c + 2] = 1 + return gal, flag, neighbour, seg, (c, c + 6) + + +def test_noisefill_interpolation_does_not_read_removed_neighbour_light(): + """Under noisefill, the interpolant of a defect beside a marked neighbour + is built from the pixels the image keeps: the neighbour's light, which + noisefill removes, is not in its support. Under uberseg the neighbour + light is raw in the image and supports the interpolant. + + Failure mode: the support includes the removed neighbour pixels, so the + defect is filled with light the image no longer contains + (interpolation-support-is-the-kept-image). + """ + gal, flag, neighbour, seg, pix = _defect_beside_neighbour() + weight = np.ones_like(gal) + target = np.zeros_like(neighbour) + target[pix] = True + + out, _, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + blend_handling="noisefill", neighbour=neighbour, + defect_fill="interpolate", + ) + expected = interpolate_defects(gal[None], (flag != 0) | neighbour, target) + assert out[pix] == pytest.approx(expected[0][pix]) + assert abs(out[pix]) < 100.0 + + out, _, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + blend_handling="uberseg", seg=seg, object_number=1, + neighbour=neighbour, defect_fill="interpolate", + ) + expected = interpolate_defects(gal[None], flag != 0, target) + assert out[pix] == pytest.approx(expected[0][pix]) + assert out[pix] > 1000.0 From b27eca849426564bee89ce814bd788db728807e0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:48:52 +0200 Subject: [PATCH 12/44] fix(ngmix): the defect fill keeps the stamp's dtype fill_defects used np.where, which promotes a float32 stamp to float64; develop's noisefill kept the galaxy stamp's dtype. The filled image is cast back to the input dtype, so noisefill stays bit-identical to develop on float32 stamps. The develop-equivalence test gains a float32 case and checks dtypes. Co-Authored-By: Claude Opus 5.5 --- src/shapepipe/modules/ngmix_package/ngmix.py | 5 +++-- tests/module/test_ngmix_defect_fill.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index bf9bf0b12..8f0ef66e1 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -2204,9 +2204,10 @@ def fill_defects(image, defect, noise): Returns ------- numpy.ndarray - ``image`` with ``defect`` pixels taken from ``noise``. + ``image`` with ``defect`` pixels taken from ``noise``, in the dtype + of ``image``. """ - return np.where(defect, noise, image) + return np.where(defect, noise, image).astype(image.dtype, copy=False) def prepare_ngmix_weights( diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 67aff0d20..17f04a2a8 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -934,13 +934,16 @@ def _develop_noisefill(gal, weight, flag, rng, bkg_rms=None): with_rms=st.booleans(), rms_scale=st.floats(0.5, 2.0), with_defects=st.booleans(), + dtype=st.sampled_from([np.float64, np.float32]), ) def test_noisefill_matches_develop_on_marked_neighbours( - seed, with_rms, rms_scale, with_defects, + seed, with_rms, rms_scale, with_defects, dtype, ): """For an epoch with a marked neighbour, noisefill returns the image, weight and noise image develop returned, bit for bit: with no defect, - and with a flagged column and a dead pixel under the default noise fill. + and with a flagged column and a dead pixel under the default noise fill, + for float64 and float32 stamps (the filled image keeps the stamp's + dtype). The stamps carry no off-tile pixels, and the equivalence is claimed for such stamps only. Off-tile pixels reach this function as flag 2**10 defects (off-tile-pixels-are-defects), as every marker did on develop; @@ -963,6 +966,7 @@ def test_noisefill_matches_develop_on_marked_neighbours( if with_defects: flag[:, 2] = 1 weight[n - 3, n // 2] = 0.0 + gal = gal.astype(dtype) bkg_rms = ( rms_scale * (1.0 + 0.1 * rng.random((n, n))) if with_rms else None ) @@ -976,6 +980,7 @@ def test_noisefill_matches_develop_on_marked_neighbours( np.random.RandomState(seed), bkg_rms=bkg_rms, ) for a, b in zip(new, old): + assert a.dtype == b.dtype npt.assert_array_equal(a, b) From a5bfef10ca0fb905d1aa5658295208d57f573ff9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 02:01:47 +0200 Subject: [PATCH 13/44] fix(ngmix): off-tile pixels are the marked rows and columns at the stamp border Every entirely -1e30 stamp row or column counted as off-tile. In an edge stamp, a neighbour footprint that covers the on-image part of a row (a wide DR6 footprint running from the tile edge) completes that row, so the neighbour's pixels became OFF_TILE_FLAG defects: counted by the epoch cuts, interpolated under DEFECT_FILL = interpolate, filled under uberseg. Only the runs of marked rows and columns that start at a stamp border are off-tile now. Tests build tile VIGNETs with the catalogue-mode converter: on the real 202.301 DR6 patch every stamp splits exactly into off-image pixels and seg-map neighbour pixels, and the edge case above keeps its footprint in the neighbour mask through prepare_postage_stamps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BbT81hykcvtuefzB3FhYvY --- astra.yaml | 17 ++-- src/shapepipe/modules/ngmix_package/ngmix.py | 30 +++++-- tests/module/test_ngmix_defect_fill.py | 91 ++++++++++++++++++++ 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/astra.yaml b/astra.yaml index 8eea6bc4c..f83377c80 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1386,13 +1386,16 @@ analyses: FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise - scale otherwise. SExtractor writes -1e30 into the tile VIGNET on - other detections' footprints and beyond the tile's edge. The stamp - rows and columns that are entirely -1e30 (the off-image part of a - rectangle clip) are off-tile: the epoch holds the object's own light - there, so they are defects, flagged 2**10. The other markers are - neighbour pixels, not defects; blend_handling decides their - treatment. The committed fill + scale otherwise. The tile VIGNET holds -1e30 on other detections' + footprints and beyond the tile's edge (written by SExtractor, or by + the unions_catalogue converter from the catalogue's segmentation + map; catalogue_neighbour_marking). The runs of entirely -1e30 stamp + rows and columns that start at a stamp border (the off-image part of + a rectangle clip) are off-tile: the epoch holds the object's own + light there, so they are defects, flagged 2**10. The other markers, + including a neighbour footprint that completes an interior row + beside an off-tile band, are neighbour pixels, not defects; + blend_handling decides their treatment. The committed fill uses the unsymmetrized defect set: DES symmetrized its masks, but four-fold symmetrization quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 2f172b4a9..4908fcd20 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -1754,13 +1754,17 @@ def prepare_postage_stamps( def split_tile_markers(tile_vign, shape): """Split the tile VIGNET's -1e30 markers into neighbour and off-tile. - @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill] off-tile-is-whole-marked-rows-and-columns - SExtractor writes -1e30 on the footprints of other detections and on - stamp pixels beyond the tile's edge. A stamp clipped by the tile's - rectangle loses whole rows and whole columns, so the off-tile pixels are - the union of the stamp rows and columns that are entirely -1e30. The - remaining markers are neighbour pixels; a footprint touching the stamp - border stays a neighbour unless it fills a whole row or column. + @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill] off-tile-is-marked-border-rows-and-columns + The tile VIGNET holds -1e30 on the footprints of other detections and on + stamp pixels beyond the tile's edge; SExtractor writes it, and in + catalogue mode the converter paints it from the catalogue's segmentation + map (``read_ext_sexcat._extract_vignets``). A stamp clipped by the tile's + rectangle loses whole rows and whole columns from its border, so the + off-tile pixels are the union of the runs of entirely -1e30 rows and + columns that start at a stamp border. The remaining markers are + neighbour pixels: a footprint touching the stamp border, and a footprint + that completes an interior row or column beside an off-tile band, stay + neighbours. Parameters ---------- @@ -1779,7 +1783,17 @@ def split_tile_markers(tile_vign, shape): if tile_vign is None: return np.zeros(shape, dtype=bool), np.zeros(shape, dtype=bool) marker = tile_vign == -1e30 - off_tile = marker.all(axis=1)[:, None] | marker.all(axis=0)[None, :] + + def border_runs(full): + # Lines in the unbroken run of marked lines from either border. + lead = np.logical_and.accumulate(full) + trail = np.logical_and.accumulate(full[::-1])[::-1] + return lead | trail + + off_tile = ( + border_runs(marker.all(axis=1))[:, None] + | border_runs(marker.all(axis=0))[None, :] + ) return marker & ~off_tile, off_tile diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 17f04a2a8..8919453c2 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -16,6 +16,7 @@ """ import re +from pathlib import Path from types import SimpleNamespace import numpy as np @@ -41,8 +42,10 @@ make_ngmix_observation, prepare_ngmix_weights, prepare_postage_stamps, + split_tile_markers, uberseg_weight, ) +from shapepipe.modules.read_ext_sexcat_package import read_ext_sexcat # --- prepare_ngmix_weights: the filled set is the defect set --------------- @@ -1124,6 +1127,94 @@ def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): npt.assert_array_equal(w_out == 0.0, removed) +# --- Catalogue mode: converter-built tile VIGNETs --------------------------- +# +# In catalogue mode the tile VIGNET comes from read_ext_sexcat, which paints +# -1e30 off the image and on other footprints of the DR6 segmentation map. +# The off-image pixels are the off-tile defects and the footprint pixels are +# the neighbour mask, exactly. + +DR6_PATCH = Path(__file__).parent / "data" / "dr6_202.301_seg_patch.fits" + + +def _converter_stamps(seg, number, x, y): + """Converter tile VIGNETs on a unit image, and each stamp's off-image + mask.""" + relabelled, _ = read_ext_sexcat.relabel_seg(seg, number, x, y) + vignets = read_ext_sexcat._extract_vignets( + np.ones(seg.shape, np.float32), x, y, N_STAMP, seg=relabelled, + number=number, + ) + half = N_STAMP // 2 + ny, nx = seg.shape + off_image = [] + for xi, yi in zip(x, y): + rows = int(np.rint(yi)) - 1 - half + np.arange(N_STAMP) + cols = int(np.rint(xi)) - 1 - half + np.arange(N_STAMP) + off_image.append( + ((rows < 0) | (rows >= ny))[:, None] + | ((cols < 0) | (cols >= nx))[None, :] + ) + return vignets, off_image + + +def test_dr6_converter_stamps_split_into_off_image_and_neighbours(): + """On the real 202.301 patch, every converter stamp splits into its + off-image pixels (off-tile) and its other -1e30 pixels (neighbours). + + Failure modes: the converter's float32 -1e30 is not recognised as a + marker; off-image pixels of an edge stamp land in the neighbour mask; + neighbour-footprint pixels become off-tile defects. + """ + with fits.open(DR6_PATCH) as hdul: + seg = hdul["SEG"].data + objects = hdul["OBJECTS"].data + number = np.array(objects["NUMBER"]) + x, y = np.array(objects["X_IMAGE"]), np.array(objects["Y_IMAGE"]) + vignets, off_image = _converter_stamps(seg, number, x, y) + assert vignets.dtype == np.float32 + n_edge = 0 + for vign, off in zip(vignets, off_image): + neighbour, off_tile = split_tile_markers(vign, vign.shape) + npt.assert_array_equal(off_tile, off) + npt.assert_array_equal(neighbour, (vign == _MARKER) & ~off) + n_edge += off.any() + assert n_edge >= 5 + assert sum( + split_tile_markers(v, v.shape)[0].sum() for v in vignets + ) > 1000 + + +def test_a_neighbour_completing_rows_beside_the_tile_edge_stays_a_neighbour(): + """An object 14 px from the tile's left edge, with a wide neighbour + footprint that runs from the tile edge across the stamp: in the rows of + that footprint every stamp pixel is -1e30, off the image or on the + neighbour. Only the off-image columns are off-tile; the footprint is the + neighbour mask, through prepare_postage_stamps. + + Failure mode: every entirely -1e30 row counts as off-tile, so the + neighbour's rows become defects that the epoch cuts count and the defect + fill interpolates (off-tile-is-marked-border-rows-and-columns). + """ + seg = np.zeros((80, 80), np.int32) + seg[20:24, 0:45] = 5 + seg[27:32, 13:18] = 1 + number, x, y = np.array([1, 5]), np.array([15.0, 30.0]), np.array([30.0, 22.0]) + vignets, off_image = _converter_stamps(seg, number, x, y) + tile, off = vignets[0], off_image[0] + footprint = (tile == _MARKER) & ~off + assert footprint.sum() == 4 * (N_STAMP - 11) + assert ((tile == _MARKER).all(axis=1) & ~off.all(axis=1)).sum() == 4 + + stamp, epochs, _ = _marker_stamp(tile) + names = {id(v[0]): name for name, v in epochs.items()} + assert len(stamp.flags) == len(_MARKER_EPOCH_NAMES) + for flag, neighbour in zip(stamp.flags, stamp.neighbours): + ccd = int(names[id(flag)].split("-")[1]) + npt.assert_array_equal(flag == _OFF_TILE, Ngmix.MegaCamFlip(off, ccd)) + npt.assert_array_equal(neighbour, Ngmix.MegaCamFlip(footprint, ccd)) + + # --- DEFECT_FILL = interpolate beside a removed neighbour ------------------- def _defect_beside_neighbour(): From 633425d94973521b1f1f7627cc69116b2e23fcc2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 02:02:35 +0200 Subject: [PATCH 14/44] docs: -1e30 tile-VIGNET markers come from SExtractor or the catalogue converter The neighbour-marking rationale and the converter's BIG comment describe the markers as ngmix neighbours under flag 2**10; ngmix splits them into off-tile defects and a neighbour mask. The masking text names the tile VIGNET, not SExtractor, as the markers' carrier, since catalogue mode paints them from the DR6 segmentation map. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BbT81hykcvtuefzB3FhYvY --- astra.yaml | 22 ++++++++++--------- src/shapepipe/modules/ngmix_package/ngmix.py | 18 +++++++-------- src/shapepipe/modules/ngmix_runner.py | 2 +- .../read_ext_sexcat.py | 5 +++-- tests/module/test_ngmix_defect_fill.py | 11 +++++----- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/astra.yaml b/astra.yaml index f83377c80..bf019aa51 100644 --- a/astra.yaml +++ b/astra.yaml @@ -637,9 +637,10 @@ analyses: catalogue_neighbour_marking: label: Neighbour pixels in the catalogue path's VIGNET rationale: >- - ngmix masks a neighbour only where the tile VIGNET is -1e30 (flag - 2**10: zero weight, noise-filled under noisefill), and SExtractor - writes -1e30 on neighbours' footprints and off the image. The + ngmix's noisefill masks a neighbour only where the tile VIGNET is + -1e30 (zero weight, noise-filled; the off-tile rows and columns are + defects instead), and SExtractor writes -1e30 on neighbours' + footprints and off the image. The unions_catalogue converter reproduces that from the catalogue's own r-band segmentation map (CFIS..r.seg.fits.fz, on the tile's pixel grid), so both tile_detection options mask the same way. @@ -1457,9 +1458,10 @@ analyses: depend on it: defects are filled the same way under every BLEND_HANDLING, and the epoch cuts never count neighbour pixels. noisefill (the default; the committed config sets no key) gives - weight 0 to the pixels SExtractor marks -1e30 in the tile VIGNET - on other detections' footprints (about 93% of neighbour-footprint - pixels on a SExtractor-mode sims tile), excluding the off-tile + weight 0 to the pixels marked -1e30 in the tile VIGNET on other + detections' footprints (about 93% of neighbour-footprint pixels on + a SExtractor-mode sims tile; every one in catalogue mode, from the + segmentation map), excluding the off-tile rows and columns, which are defects, and replaces them with noise; unmarked neighbour pixels keep their weight and light. uberseg ignores the @@ -1487,7 +1489,7 @@ analyses: default: noisefill options: noisefill: - label: Noise-fill SExtractor-marked neighbour pixels (BLEND_HANDLING = noisefill) + label: Noise-fill the marked neighbour pixels (BLEND_HANDLING = noisefill) description: >- Pixels marked -1e30 in the tile VIGNET get weight 0 and noise. The fill stops at the marked footprint, so unmarked neighbour @@ -1527,7 +1529,7 @@ analyses: An epoch is dropped when a defect pixel lies strictly closer to the stamp centre than its fill's radius, beside the masked-fraction cut in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. - SExtractor's -1e30 neighbour markers are not defects: all epochs + The tile VIGNET's -1e30 neighbour markers are not defects: all epochs share the tile VIGNET, so a neighbour inside the radius would drop every epoch. Off-tile pixels are defects, so an object near the tile edge is vetoed. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same @@ -1566,8 +1568,8 @@ analyses: (default 1/3) of its pixels in the raw, unsymmetrized defect set is dropped from the multi-epoch fit: flagged pixels (any nonzero exposure flag bit), zero-weight pixels, invalid-RMS pixels and - off-tile pixels (whole -1e30 rows and columns of the tile VIGNET, - flag 2**10), the set defect_fill fills. On a 51-px stamp an object + off-tile pixels (whole -1e30 rows and columns at the tile VIGNET's + border, flag 2**10), the set defect_fill fills. On a 51-px stamp an object within about 8.5 px of the tile edge fails the 1/3 cut. The other -1e30 markers, neighbour footprints, are not counted: every epoch shares the tile VIGNET, so a large neighbour would drop them all. An object with no surviving epoch has no diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 4908fcd20..471e62b2c 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -633,8 +633,8 @@ def __init__( self.psfs = [] self.weights = [] self.flags = [] - # Neighbour masks, one per epoch: the pixels SExtractor marks -1e30 in - # the tile VIGNET on other detections' footprints (off-tile markers + # Neighbour masks, one per epoch: the pixels marked -1e30 in the tile + # VIGNET on other detections' footprints (off-tile markers # are flagged as defects instead; see split_tile_markers), # MegaCam-flipped to the epoch. noisefill zero-weights and noise-fills them; uberseg and # the epoch cuts do not read them (see prepare_ngmix_weights). @@ -791,8 +791,8 @@ class Ngmix(object): :func:`make_ngmix_observation`. blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment. ``"noisefill"`` (default) zero-weights and - noise-fills the pixels SExtractor marks -1e30 in the tile VIGNET - on other detections' footprints; ``"uberseg"`` ignores those markers, + noise-fills the pixels marked -1e30 in the tile VIGNET on other + detections' footprints; ``"uberseg"`` ignores those markers, zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect pixels are filled under both (see :func:`prepare_ngmix_weights`). @@ -1555,8 +1555,8 @@ def prepare_postage_stamps( 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects - SExtractor writes -1e30 into the tile VIGNET on the footprints of other - detections and beyond the tile's edge (:func:`split_tile_markers`). The + The tile VIGNET holds -1e30 on the footprints of other detections and + beyond the tile's edge (:func:`split_tile_markers`). The footprint markers form the epoch's neighbour mask (``stamp.neighbours``, MegaCam-flipped like the epoch), kept apart from its flag stamp, so neither the masked-fraction cut nor the central veto counts them. Every @@ -2089,7 +2089,7 @@ def defect_mask(weight, flag, bkg_rms=None): or (when a background RMS map is given) a non-finite or non-positive RMS. This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` under every ``blend_handling`` and counted by the epoch cuts in - :func:`prepare_postage_stamps`. SExtractor's neighbour markers are not + :func:`prepare_postage_stamps`. The tile VIGNET's neighbour markers are not in it (neighbour-markers-are-not-defects); off-tile pixels are, as flag ``OFF_TILE_FLAG`` (off-tile-pixels-are-defects). It is not ORed with its rotations. For defects the central-defect veto keeps @@ -2280,8 +2280,8 @@ def prepare_ngmix_weights( supports the interpolant. @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers - Under ``"noisefill"`` the pixels of ``neighbour``, SExtractor's -1e30 - neighbour markers, get weight 0 and are replaced by the same noise + Under ``"noisefill"`` the pixels of ``neighbour``, the tile VIGNET's + -1e30 neighbour markers, get weight 0 and are replaced by the same noise realisation as the noise-filled defects, so no marked neighbour light reaches metacal. Under the default noise fill, the image, weight map and noise image are those the marked pixels would get as flagged defects; diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 205a38cab..08113aa84 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -145,7 +145,7 @@ def ngmix_runner( centroid_source = "wcs" # Neighbour treatment: "noisefill" (default) zero-weights and noise-fills - # the pixels SExtractor marks -1e30 in the tile VIGNET (other detections' + # the pixels marked -1e30 in the tile VIGNET (other detections' # footprints); "uberseg" ignores those markers, zeroes the weight of every # pixel closer to a neighbour than to the central object, from the # segmentation map, and leaves its image raw. Defect pixels (flagged, diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py index ff4f58cc8..4b1d66012 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py @@ -48,8 +48,9 @@ def _build_ldac_imhead(img_header): # SExtractor's VIGNET value for pixels that are not the object's: off the -# image and on a neighbour's segmentation footprint. ngmix flags these pixels -# (tile VIGNET == -1e30) and noise-fills them at zero weight. +# image and on a neighbour's segmentation footprint. ngmix splits these pixels +# (tile VIGNET == -1e30, ``ngmix.split_tile_markers``): the off-image rows and +# columns are defects, the footprint pixels its neighbour mask. BIG = -1e30 # The label of a footprint no catalogue object claims in the relabelled diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 8919453c2..f068c1290 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -10,7 +10,7 @@ in :func:`prepare_postage_stamps` act on the same defect set: the masked-fraction cut counts it, and the central-defect veto drops an epoch with a defect near the stamp centre, at a radius set by the defect's fill. -SExtractor's -1e30 neighbour markers in the tile VIGNET are not defects: +The tile VIGNET's -1e30 neighbour markers are not defects: noisefill zero-weights and noise-fills them, uberseg ignores them, and the epoch cuts never count them. """ @@ -744,7 +744,7 @@ def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): ) -# --- SExtractor's -1e30 neighbour markers are not defects ------------------- +# --- The tile VIGNET's -1e30 neighbour markers are not defects ------------- # # The tile VIGNET carries -1e30 on the footprints of other detections. Every # epoch shares that tile stamp, so a marker counted as a defect would drop @@ -1021,9 +1021,10 @@ def fake_observation(*args, **kwargs): # --- Off-tile pixels are defects ------------------------------------------- # -# SExtractor also writes -1e30 beyond the tile's edge, where the epoch holds -# the object's own light, cut off. Those pixels are the stamp rows and -# columns that are entirely -1e30 (the off-image part of a rectangle clip); +# The tile VIGNET also holds -1e30 beyond the tile's edge, where the epoch +# holds the object's own light, cut off. Those pixels are the runs of +# entirely -1e30 stamp rows and columns that start at a stamp border (the +# off-image part of a rectangle clip); # they join the epoch's defect set as flag 2**10. The other markers are the # neighbour mask. From d35af2eaf373f55bdb59ef1d6d7a3023265d9186 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 00:27:12 +0200 Subject: [PATCH 15/44] Expand ${VAR:-default} in config values A committed ini can then name a workflow knob with its default, and the workflow exports the variable only when the knob leaves its default. Co-Authored-By: Claude Opus 5.5 --- src/shapepipe/pipeline/config.py | 53 ++++++++++++++++++++++++++++---- tests/module/test_pipeline.py | 40 ++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/src/shapepipe/pipeline/config.py b/src/shapepipe/pipeline/config.py index 4e809316b..894c60a4a 100644 --- a/src/shapepipe/pipeline/config.py +++ b/src/shapepipe/pipeline/config.py @@ -11,18 +11,24 @@ from configparser import ConfigParser +# ``$VAR``, ``${VAR}`` or ``${VAR:-default}``. +_VAR_RE = re.compile(r"\$(?:\{(\w+)(?::-([^}]*))?\}|(\w+))") + + def _expandvars_strict(value): """Expand Environment Variables Strictly. Expand environment variables in ``value``, raising an error if any referenced variable is unset instead of silently leaving the literal - ``$VAR`` string in place. + ``$VAR`` string in place. ``${VAR:-default}`` expands to ``default`` + when ``VAR`` is unset or empty, as in the shell. Each value is inserted + as is, not expanded again. Parameters ---------- value : str - Configuration value possibly containing ``$VAR`` or ``${VAR}`` - references + Configuration value possibly containing ``$VAR``, ``${VAR}`` or + ``${VAR:-default}`` references Returns ------- @@ -32,11 +38,23 @@ def _expandvars_strict(value): Raises ------ ValueError - If a referenced environment variable is not set + If a referenced environment variable without a default is not set """ - expanded = os.path.expandvars(value) - unset = re.findall(r"\$\{?(\w+)\}?", expanded) + unset = [] + + def substitute(match): + braced, default, bare = match.groups() + name = braced or bare + env = os.environ.get(name) + if default is not None: + return env or default + if env is None: + unset.append(name) + return match.group(0) + return env + + expanded = _VAR_RE.sub(substitute, value) if unset: raise ValueError( f"Environment variable(s) {', '.join(sorted(set(unset)))} " @@ -72,6 +90,29 @@ def getexpanded(self, section, option, **kwargs): """ return self._get(section, _expandvars_strict, option, **kwargs) + def getexpandedboolean(self, section, option, **kwargs): + """Get Expanded Boolean. + + Expand enviroment variables in the value, then read it as a boolean + the way ``getboolean`` does. + + Parameters + ---------- + section : str + Configuration file section + option : str + Configuration file option + + Returns + ------- + bool + The expanded value as a boolean + + """ + return self._convert_to_boolean( + self.getexpanded(section, option, **kwargs) + ) + def getlist(self, section, option, delimiter=",", **kwargs): """Get List. diff --git a/tests/module/test_pipeline.py b/tests/module/test_pipeline.py index 400ca2ec3..125f24dc6 100644 --- a/tests/module/test_pipeline.py +++ b/tests/module/test_pipeline.py @@ -134,6 +134,46 @@ def test_custom_parser_getexpanded_expands_fallback(monkeypatch): ) +def test_custom_parser_expands_a_default_only_when_the_variable_is_unset( + monkeypatch, +): + """``${VAR:-default}`` takes ``default`` when VAR is unset or empty, and + VAR's value otherwise; a bare unset ``$VAR`` still raises.""" + parser = config.CustomParser() + parser.add_section("S") + parser.set("S", "MODE", "${SP_TEST_MODE:-noisefill}") + parser.set("S", "FLAG", "${SP_TEST_FLAG:-False}") + parser.set("S", "BARE", "$SP_TEST_MODE") + + monkeypatch.delenv("SP_TEST_MODE", raising=False) + monkeypatch.delenv("SP_TEST_FLAG", raising=False) + assert parser.getexpanded("S", "MODE") == "noisefill" + assert parser.getexpandedboolean("S", "FLAG") is False + with pytest.raises(ValueError, match="SP_TEST_MODE"): + parser.getexpanded("S", "BARE") + + monkeypatch.setenv("SP_TEST_MODE", "") + assert parser.getexpanded("S", "MODE") == "noisefill" + + monkeypatch.setenv("SP_TEST_MODE", "uberseg") + monkeypatch.setenv("SP_TEST_FLAG", "True") + assert parser.getexpanded("S", "MODE") == "uberseg" + assert parser.getexpanded("S", "BARE") == "uberseg" + assert parser.getexpandedboolean("S", "FLAG") is True + + +def test_custom_parser_expands_a_variable_value_once(monkeypatch): + """A variable's value is inserted as is, never expanded again.""" + monkeypatch.setenv("SP_TEST_ROOT", "/data/$SP_TEST_OTHER") + monkeypatch.setenv("SP_TEST_OTHER", "expanded-again") + parser = config.CustomParser() + parser.add_section("S") + parser.set("S", "PATH", "${SP_TEST_ROOT:-/x}/a $SP_TEST_ROOT/b") + assert parser.getexpanded("S", "PATH") == ( + "/data/$SP_TEST_OTHER/a /data/$SP_TEST_OTHER/b" + ) + + def test_custom_parser_getlist_honours_custom_delimiter(): parser = config.CustomParser() From 540a220dab4029ff20994a2ebc1f7a47a91db26f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 00:31:14 +0200 Subject: [PATCH 16/44] Write SEG_VIGNET from the catalogue-mode converter Under SEG_VIGNET = True the converter adds one int32 stamp per object of the relabelled segmentation map, cut with the window and centres VIGNET is cut with, so the two register pixel for pixel. The stamp cutter moves to sextractor_script.cut_stamps for the SExtractor path to share. Co-Authored-By: Claude Opus 5.5 --- .../read_ext_sexcat.py | 73 ++++++----- .../modules/read_ext_sexcat_runner.py | 14 +- .../sextractor_package/sextractor_script.py | 65 ++++++++++ tests/module/test_ngmix_defect_fill.py | 2 +- tests/module/test_read_ext_sexcat.py | 121 ++++++++++++++++-- 5 files changed, 229 insertions(+), 46 deletions(-) diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py index 4b1d66012..841f89259 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py @@ -11,6 +11,8 @@ from astropy.io import ascii as asc from astropy.io import fits +from shapepipe.modules.sextractor_package import sextractor_script as ss + def _build_ldac_imhead(img_header): """Build LDAC_IMHEAD extension from a FITS image header. @@ -55,7 +57,8 @@ def _build_ldac_imhead(img_header): # The label of a footprint no catalogue object claims in the relabelled # segmentation map. Negative, so it never collides with a NUMBER; VIGNET -# marking only asks "self or not self", so neighbours need no identity. +# marking and UberSeg only ask "self or not self", so neighbours need no +# identity. NEIGHBOUR_LABEL = -1 # Radius, in pixels, of the disc of its own NUMBER painted for an object with @@ -153,9 +156,9 @@ def _extract_vignets(image_data, x_pos, y_pos, stamp_size, seg=None, For each object position, a ``stamp_size x stamp_size`` cutout is extracted, centred on the pixel holding the position. Pixels off the image are set to ``BIG``, as SExtractor does. With a segmentation map in - the catalogue's numbering (:func:`relabel_seg`), pixels on any footprint - other than the object's own are set to ``BIG`` too, which is how - SExtractor's VIGNET marks neighbours. + the catalogue's numbering (:func:`relabel_seg`), the map is cut on the + same grid, and pixels on any footprint other than the object's own are + set to ``BIG`` too, which is how SExtractor's VIGNET marks neighbours. Parameters ---------- @@ -175,33 +178,27 @@ def _extract_vignets(image_data, x_pos, y_pos, stamp_size, seg=None, Returns ------- numpy.ndarray - Array of shape ``(n_obj, stamp_size, stamp_size)``, dtype float32 + Image stamps, shape ``(n_obj, stamp_size, stamp_size)``, float32 + numpy.ndarray or None + Segmentation stamps on the same grid, 0 off the map, or ``None`` + without ``seg`` @sc [decision:detection.catalogue_neighbour_marking] """ - ny, nx = image_data.shape - half = stamp_size // 2 col, row = _centre_pixels(x_pos, y_pos) - vignets = np.full((len(col), stamp_size, stamp_size), BIG, np.float32) - seg_stamp = np.zeros((stamp_size, stamp_size), np.int32) - - for i, (xi, yi) in enumerate(zip(col, row)): - x0, y0 = xi - half, yi - half - xc0, xc1 = max(0, x0), min(nx, x0 + stamp_size) - yc0, yc1 = max(0, y0), min(ny, y0 + stamp_size) - if xc0 >= xc1 or yc0 >= yc1: - continue - inner = np.s_[yc0 - y0:yc1 - y0, xc0 - x0:xc1 - x0] - vignets[i][inner] = image_data[yc0:yc1, xc0:xc1] - if seg is not None: - seg_stamp[:] = 0 - seg_stamp[inner] = seg[yc0:yc1, xc0:xc1] - vignets[i][(seg_stamp != 0) & (seg_stamp != number[i])] = BIG - - return vignets + vignets = ss.cut_stamps( + image_data.astype(np.float32, copy=False), col, row, stamp_size, + np.float32(BIG), + ) + if seg is None: + return vignets, None + seg_vignets = ss.cut_stamps(seg, col, row, stamp_size, 0) + own = np.asarray(number)[:, None, None] + vignets[(seg_vignets != 0) & (seg_vignets != own)] = BIG + return vignets, seg_vignets -def _build_ldac_objects(cat_data, vignets): +def _build_ldac_objects(cat_data, vignets, seg_vignets=None): """Build LDAC_OBJECTS extension from an astropy table. Parameters @@ -210,6 +207,9 @@ def _build_ldac_objects(cat_data, vignets): Catalogue data read from the ASCII SExtractor file vignets : numpy.ndarray Array of shape ``(n_obj, stamp_size, stamp_size)`` + seg_vignets : numpy.ndarray, optional + Segmentation stamps on the grid of ``vignets``, written as the + ``SEG_VIGNET`` column when given Returns ------- @@ -257,6 +257,8 @@ def _build_ldac_objects(cat_data, vignets): dim=f"({stamp_size},{stamp_size})", ) fits_cols.append(vignet_col) + if seg_vignets is not None: + fits_cols.append(ss.seg_vignet_column(seg_vignets)) hdu = fits.BinTableHDU.from_columns(fits_cols) hdu.name = "LDAC_OBJECTS" @@ -269,6 +271,7 @@ def make_ldac_from_ascii( output_cat_path, stamp_size=51, seg_path=None, + seg_vignet=False, w_log=None, ): """Convert an external ASCII catalogue to FITS-LDAC format. @@ -284,8 +287,11 @@ def make_ldac_from_ascii( ``VIGNET`` column (postage stamps extracted from the tile image) is added to ``LDAC_OBJECTS``. Given the catalogue's segmentation map, which shares the tile's pixel grid, the map is relabelled to the catalogue's - ``NUMBER`` (:func:`relabel_seg`) and used to set neighbours' pixels in each ``VIGNET`` to ``BIG``, as - SExtractor does. + ``NUMBER`` (:func:`relabel_seg`) and used to set neighbours' pixels + in each ``VIGNET`` to ``BIG``, as SExtractor does. With ``seg_vignet``, + the relabelled map's stamps, cut on each ``VIGNET``'s grid, are written + as the int32 ``SEG_VIGNET`` column that ngmix's UberSeg blend handling + reads. Parameters ---------- @@ -299,10 +305,17 @@ def make_ldac_from_ascii( Side length of the square postage stamp in pixels, default 51 seg_path : str, optional Path to the catalogue's segmentation map (FITS, compressed or not) + seg_vignet : bool, optional + Write the ``SEG_VIGNET`` column, which needs ``seg_path``; default + ``False`` w_log : logging.Logger, optional Pipeline logger """ + if seg_vignet and seg_path is None: + raise ValueError( + "SEG_VIGNET needs the catalogue's segmentation map (seg_path)." + ) cat_data = asc.read(input_cat_path, format="sextractor") n_obj = len(cat_data) if w_log: @@ -337,7 +350,7 @@ def make_ldac_from_ascii( w_log.info( f"Extracting {stamp_size}x{stamp_size} vignets from {image_path}" ) - vignets = _extract_vignets( + vignets, seg_vignets = _extract_vignets( image_data, cat_data["X_IMAGE"], cat_data["Y_IMAGE"], @@ -347,7 +360,9 @@ def make_ldac_from_ascii( ) ldac_imhead = _build_ldac_imhead(img_header) - ldac_objects = _build_ldac_objects(cat_data, vignets) + ldac_objects = _build_ldac_objects( + cat_data, vignets, seg_vignets if seg_vignet else None + ) hdul_out = fits.HDUList([fits.PrimaryHDU(), ldac_imhead, ldac_objects]) hdul_out.writeto(output_cat_path, overwrite=True) diff --git a/src/shapepipe/modules/read_ext_sexcat_runner.py b/src/shapepipe/modules/read_ext_sexcat_runner.py index 17e23163d..c6640df92 100644 --- a/src/shapepipe/modules/read_ext_sexcat_runner.py +++ b/src/shapepipe/modules/read_ext_sexcat_runner.py @@ -33,15 +33,22 @@ def read_ext_sexcat_runner( The inputs are the catalogue and the tile image, then the catalogue's segmentation map if SEGMENTATION = True, then the WCS log if MAKE_POST_PROCESS = True. With the segmentation map, neighbours' pixels - in VIGNET are set to -1e30; the map itself is not written out; the - only output is the FITS-LDAC ``sexcat.fits``. MAKE_POST_PROCESS runs the - multi-epoch post-processing that adds per-exposure HDUs. + in VIGNET are set to -1e30. SEG_VIGNET = True (it needs SEGMENTATION) + adds the map's stamps, relabelled to the catalogue's NUMBER and cut on + each VIGNET's grid, as the SEG_VIGNET column ngmix's UberSeg blend + handling reads; the value is environment-expanded. The map itself is + not written out: the only output is the FITS-LDAC + ``sexcat.fits``. MAKE_POST_PROCESS runs the multi-epoch + post-processing that adds per-exposure HDUs. """ cat_path, image_path, *extra_inputs = input_file_list use_seg = config.has_option( module_config_sec, "SEGMENTATION" ) and config.getboolean(module_config_sec, "SEGMENTATION") seg_path = extra_inputs.pop(0) if use_seg else None + seg_vignet = config.has_option( + module_config_sec, "SEG_VIGNET" + ) and config.getexpandedboolean(module_config_sec, "SEG_VIGNET") if config.has_option(module_config_sec, "SUFFIX"): suffix = config.get(module_config_sec, "SUFFIX") @@ -64,6 +71,7 @@ def read_ext_sexcat_runner( output_path, stamp_size=stamp_size, seg_path=seg_path, + seg_vignet=seg_vignet, w_log=w_log, ) diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index 8faf240ab..94b7d998d 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -16,6 +16,71 @@ from shapepipe.pipeline import file_io +def cut_stamps(array, col, row, stamp_size, fill): + """Cut one square stamp per object from a 2-D array. + + Each stamp is ``stamp_size`` pixels on a side with the 0-based pixel + (``row``, ``col``) at index ``stamp_size // 2`` on both axes; pixels off + the array take ``fill``. This is the window SExtractor cuts ``VIGNET`` + with, so stamps cut at the same centres from arrays on one pixel grid are + registered pixel for pixel. + + Parameters + ---------- + array : numpy.ndarray + 2-D array, shape ``(ny, nx)`` + col, row : array_like of int + 0-based column and row of each stamp's centre pixel + stamp_size : int + Side length of the stamps + fill : scalar + Value of the stamp pixels off the array + + Returns + ------- + numpy.ndarray + Stamps, shape ``(n_obj, stamp_size, stamp_size)``, dtype of ``array`` + + """ + ny, nx = array.shape + half = stamp_size // 2 + stamps = np.full((len(col), stamp_size, stamp_size), fill, array.dtype) + for i, (xi, yi) in enumerate(zip(col, row)): + x0, y0 = xi - half, yi - half + xc0, xc1 = max(0, x0), min(nx, x0 + stamp_size) + yc0, yc1 = max(0, y0), min(ny, y0 + stamp_size) + if xc0 >= xc1 or yc0 >= yc1: + continue + stamps[i, yc0 - y0:yc1 - y0, xc0 - x0:xc1 - x0] = ( + array[yc0:yc1, xc0:xc1] + ) + return stamps + + +def seg_vignet_column(seg_vignets): + """The ``SEG_VIGNET`` LDAC column: one int32 seg stamp per object. + + Parameters + ---------- + seg_vignets : numpy.ndarray + Segmentation stamps, shape ``(n_obj, stamp_size, stamp_size)``, cut + on ``VIGNET``'s grid (:func:`cut_stamps`) + + Returns + ------- + astropy.io.fits.Column + The column, laid out as ``VIGNET`` is + + """ + n_obj, ny, nx = seg_vignets.shape + return fits.Column( + name="SEG_VIGNET", + format=f"{ny * nx}J", + array=seg_vignets.astype(np.int32, copy=False).reshape(n_obj, -1), + dim=f"({nx},{ny})", + ) + + def get_header_value(image_path, key): """Get Header Value. diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index f068c1290..29d45f0db 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -1142,7 +1142,7 @@ def _converter_stamps(seg, number, x, y): """Converter tile VIGNETs on a unit image, and each stamp's off-image mask.""" relabelled, _ = read_ext_sexcat.relabel_seg(seg, number, x, y) - vignets = read_ext_sexcat._extract_vignets( + vignets, _ = read_ext_sexcat._extract_vignets( np.ones(seg.shape, np.float32), x, y, N_STAMP, seg=relabelled, number=number, ) diff --git a/tests/module/test_read_ext_sexcat.py b/tests/module/test_read_ext_sexcat.py index fa406b9a6..44dee1a01 100644 --- a/tests/module/test_read_ext_sexcat.py +++ b/tests/module/test_read_ext_sexcat.py @@ -230,16 +230,22 @@ def _marked(number, x, y, seg=None): image = np.ones(_seg_map().shape, np.float32) seg = _seg_map() if seg is None else seg relabelled, _ = rs.relabel_seg(seg, number, x, y) - return rs._extract_vignets(image, x, y, SEG_STAMP, seg=relabelled, - number=number) + vignets, _ = rs._extract_vignets(image, x, y, SEG_STAMP, seg=relabelled, + number=number) + return vignets + + +def _stamp_at(array, x, y, size, fill): + """The ``size`` stamp of ``array`` centred on 1-based (x, y).""" + half = size // 2 + padded = np.pad(array, half, constant_values=fill) + row, col = int(np.rint(y)) - 1, int(np.rint(x)) - 1 + return padded[row:row + size, col:col + size] def _stamp_of(array, x, y, fill): """The SEG_STAMP stamp of ``array`` centred on 1-based (x, y).""" - half = SEG_STAMP // 2 - padded = np.pad(array, half, constant_values=fill) - row, col = int(round(y)) - 1, int(round(x)) - 1 - return padded[row:row + SEG_STAMP, col:col + SEG_STAMP] + return _stamp_at(array, x, y, SEG_STAMP, fill) @pytest.mark.decision("detection.catalogue_neighbour_marking") @@ -346,8 +352,8 @@ def test_dr6_marks_every_neighbour_pixel_and_no_own_pixel(): stamp, half = 51, 25 image = np.ones(seg.shape, np.float32) - vignets = rs._extract_vignets(image, x, y, stamp, seg=relabelled, - number=number) + vignets, seg_vignets = rs._extract_vignets(image, x, y, stamp, + seg=relabelled, number=number) col, row = np.rint(x).astype(int) - 1, np.rint(y).astype(int) - 1 full = ((col >= half) & (col < seg.shape[1] - half) & (row >= half) & (row < seg.shape[0] - half)) @@ -366,23 +372,109 @@ def test_dr6_marks_every_neighbour_pixel_and_no_own_pixel(): assert marked["neighbour"][0] == marked["neighbour"][1] assert marked["own"][0] == 0 assert marked["sky"][0] == 0 + # SEG_VIGNET is the relabelled map on VIGNET's grid, and VIGNET's + # neighbour marks are exactly its foreign labels. + for i in range(len(number)): + npt.assert_array_equal(seg_vignets[i], + _stamp_at(relabelled, x[i], y[i], stamp, 0)) + on_image = _stamp_at(np.ones(seg.shape, bool), x[i], y[i], stamp, + False) + foreign = (seg_vignets[i] != 0) & (seg_vignets[i] != number[i]) + npt.assert_array_equal((vignets[i] == rs.BIG) & on_image, foreign) + + +# --- SEG_VIGNET: the coadd seg stamp UberSeg reads ------------------------ + + +def _convert_with_seg(tmp_path, seg_vignet, objects=SEG_OBJECTS): + tmp_path.mkdir(exist_ok=True) + cat = tmp_path / "CFIS_cat-301-279.cat" + img = tmp_path / "CFIS_image-301-279.fits" + seg_in = tmp_path / "CFIS_seg-301-279.fits" + out = tmp_path / "sexcat-301-279.fits" + lines = ["# 1 NUMBER", "# 2 X_IMAGE", "# 3 Y_IMAGE", + "# 4 ALPHA_J2000", "# 5 DELTA_J2000"] + lines += [f"{n} {x} {y} 150.0 30.0" for n, x, y in objects] + cat.write_text("\n".join(lines) + "\n") + image = np.arange(400, dtype=np.float32).reshape(20, 20) + fits.PrimaryHDU(image).writeto(img, overwrite=True) + fits.PrimaryHDU(_seg_map()).writeto(seg_in, overwrite=True) + rs.make_ldac_from_ascii(str(cat), str(img), str(out), stamp_size=SEG_STAMP, + seg_path=str(seg_in), seg_vignet=seg_vignet) + with fits.open(out) as hdul: + data = hdul["LDAC_OBJECTS"].data + number, x, y = (np.array(c) for c in zip(*objects)) + relabelled, _ = rs.relabel_seg(_seg_map(), number, x, y) + return data, relabelled + + +def test_seg_vignet_is_written_only_when_asked(tmp_path): + """The column exists only under SEG_VIGNET, and asking for it changes no + other column: a noise-fill sexcat is the sexcat without it.""" + without, _ = _convert_with_seg(tmp_path / "a", seg_vignet=False) + with_, _ = _convert_with_seg(tmp_path / "b", seg_vignet=True) + assert "SEG_VIGNET" not in without.names + assert with_.names == without.names + ["SEG_VIGNET"] + for name in without.names: + npt.assert_array_equal(with_[name], without[name]) + + +# Positions on exact half pixels round the way VIGNET does (np.rint). +REGISTRATION_OBJECTS = SEG_OBJECTS + [(3, 13.5, 14.5), (4, 1.0, 19.5), + (5, 20.0, 1.0)] + + +def test_seg_vignet_is_registered_with_vignet(tmp_path): + """SEG_VIGNET is the relabelled map cut on VIGNET's own grid: int32, the + same shape, 0 off the image, and VIGNET's on-image -1e30 pixels are + exactly its foreign labels, so the two agree pixel for pixel.""" + data, relabelled = _convert_with_seg(tmp_path / "c", seg_vignet=True, + objects=REGISTRATION_OBJECTS) + seg_vignets, vignets = data["SEG_VIGNET"], data["VIGNET"] + assert seg_vignets.dtype.kind == "i" and seg_vignets.dtype.itemsize == 4 + assert seg_vignets.shape == vignets.shape + image = np.arange(400, dtype=np.float32).reshape(20, 20) + for i, (number, x, y) in enumerate(REGISTRATION_OBJECTS): + npt.assert_array_equal(seg_vignets[i], + _stamp_of(relabelled, x, y, fill=0)) + on_image = _stamp_of(np.ones((20, 20), bool), x, y, fill=False) + foreign = (seg_vignets[i] != 0) & (seg_vignets[i] != number) + npt.assert_array_equal((vignets[i] == rs.BIG) & on_image, foreign) + clean = on_image & ~foreign + npt.assert_array_equal(vignets[i][clean], + _stamp_of(image, x, y, fill=0)[clean]) + assert seg_vignets[i][SEG_STAMP // 2, SEG_STAMP // 2] == number + + +def test_seg_vignet_needs_the_segmentation_map(tmp_path): + cat = tmp_path / "CFIS_cat-301-279.cat" + img = tmp_path / "CFIS_image-301-279.fits" + _write_ascii_cat(cat) + _write_image(img) + with pytest.raises(ValueError, match="SEG_VIGNET"): + rs.make_ldac_from_ascii(str(cat), str(img), + str(tmp_path / "sexcat-301-279.fits"), + stamp_size=STAMP, seg_vignet=True) # --- the runner's output against the completeness table ------------------- -def test_runner_output_matches_tile_detect_completeness(tmp_path, monkeypatch): +@pytest.mark.parametrize("seg_vignet", ["False", "True"]) +def test_runner_output_matches_tile_detect_completeness(tmp_path, monkeypatch, + seg_vignet): """The runner writes exactly the files ``tile_detect`` expects. Runs the real runner, segmentation map on, into a run dir and checks it with ``completeness.check_counts`` under ``SP_TILE_DETECTION=unions_catalogue``, - so the table and the converter's outputs cannot drift apart. + so the table and the converter's outputs cannot drift apart. SEG_VIGNET + is a column of the sexcat, so it adds no file. """ - import configparser import importlib.util import logging from shapepipe.modules.read_ext_sexcat_runner import read_ext_sexcat_runner + from shapepipe.pipeline.config import CustomParser scripts = Path(__file__).resolve().parents[2] / "workflow" / "scripts" spec = importlib.util.spec_from_file_location( @@ -404,10 +496,10 @@ def test_runner_output_matches_tile_detect_completeness(tmp_path, monkeypatch): run_dir = tmp_path / "run_sp_tile_Rx" out_dir = run_dir / "read_ext_sexcat_runner" / "output" out_dir.mkdir(parents=True) - config = configparser.ConfigParser() + config = CustomParser() config["READ_EXT_SEXCAT_RUNNER"] = { "SEGMENTATION": "True", "MAKE_POST_PROCESS": "False", - "VIGNET_SIZE": str(SEG_STAMP), + "VIGNET_SIZE": str(SEG_STAMP), "SEG_VIGNET": seg_vignet, } read_ext_sexcat_runner( [str(cat), str(img), str(seg_in)], {"output": str(out_dir)}, @@ -422,3 +514,6 @@ def test_runner_output_matches_tile_detect_completeness(tmp_path, monkeypatch): assert len(written) == expect, written ok, details = completeness.check_counts("tile_detect", run_dir) assert ok, details + with fits.open(out_dir / written[0]) as hdul: + names = hdul["LDAC_OBJECTS"].data.names + assert ("SEG_VIGNET" in names) == (seg_vignet == "True") From 13f02c238129900919ab63b48eab9235f0a73ab1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 00:33:48 +0200 Subject: [PATCH 17/44] Write SEG_VIGNET from the tile SExtractor run Under SEG_VIGNET = True, sextractor_runner cuts the SEGMENTATION check image on each object's VIGNET grid and adds it to the sexcat. VIGNET is centred on the pixel nearest SExtractor's double barycentre; the float32 X_IMAGE/Y_IMAGE round to it except on exact half pixels, which are resolved by matching VIGNET against image minus BACKGROUND. Co-Authored-By: Claude Opus 5.5 --- .../sextractor_package/sextractor_script.py | 121 +++++++++++++++ src/shapepipe/modules/sextractor_runner.py | 21 +++ tests/module/test_sextractor_seg_vignet.py | 140 ++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 tests/module/test_sextractor_seg_vignet.py diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index 94b7d998d..854872e9c 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -81,6 +81,125 @@ def seg_vignet_column(seg_vignets): ) +def vignet_centres(x_image, y_image, vignets, image, background): + """0-based centre pixels of SExtractor's VIGNET stamps. + + SExtractor centres VIGNET on the pixel nearest the object's barycentre, + held in double precision. The catalogue's float32 ``X_IMAGE`` / + ``Y_IMAGE`` round to the same pixel except where they land on an exact + half pixel, which hides the side the barycentre was on. For those, each + neighbouring centre is tried and the one whose stamp of ``image`` minus + ``background`` agrees with the object's VIGNET on the most pixels wins. + + Parameters + ---------- + x_image, y_image : numpy.ndarray + 1-based positions as the catalogue stores them + vignets : numpy.ndarray + The catalogue's VIGNET stamps, shape ``(n_obj, size, size)`` + image, background : numpy.ndarray + The measurement image and SExtractor's BACKGROUND check image + + Returns + ------- + numpy.ndarray + 0-based column of each stamp's centre pixel + numpy.ndarray + 0-based row of each stamp's centre pixel + int + Number of objects on a half pixel + + """ + x = np.asarray(x_image, dtype=np.float64) - 1 + y = np.asarray(y_image, dtype=np.float64) - 1 + col, row = np.rint(x).astype(np.int64), np.rint(y).astype(np.int64) + x_tie, y_tie = x % 1 == 0.5, y % 1 == 0.5 + size = vignets.shape[1] + image = image.astype(np.float32, copy=False) + background = background.astype(np.float32, copy=False) + ties = np.flatnonzero(x_tie | y_tie) + for i in ties: + cols = np.floor(x[i]) + np.array([0, 1]) if x_tie[i] else [col[i]] + rows = np.floor(y[i]) + np.array([0, 1]) if y_tie[i] else [row[i]] + candidates = [(int(c), int(r)) for c in cols for r in rows] + valid = vignets[i] > -1e29 + + def agreement(centre): + c, r = [centre[0]], [centre[1]] + stamp = (cut_stamps(image, c, r, size, np.float32(0))[0] + - cut_stamps(background, c, r, size, np.float32(0))[0]) + return np.count_nonzero((stamp == vignets[i]) & valid) + + col[i], row[i] = max(candidates, key=agreement) + return col, row, len(ties) + + +def add_seg_vignet(cat_path, seg_path, image_path, background_path, + w_log=None): + """Add the ``SEG_VIGNET`` column to a SExtractor catalogue. + + The SEGMENTATION check image, whose labels are the catalogue's + ``NUMBER``, is cut on the grid of each object's VIGNET + (:func:`vignet_centres`, :func:`cut_stamps`) and written, int32 and 0 + off the image, as ``SEG_VIGNET`` in ``LDAC_OBJECTS``, which ngmix's + UberSeg blend handling reads. Every other HDU and column is kept. + + Parameters + ---------- + cat_path : str + Path to the SExtractor FITS-LDAC catalogue, rewritten in place + seg_path : str + Path to the SEGMENTATION check image + image_path : str + Path to the measurement image + background_path : str + Path to the BACKGROUND check image + w_log : logging.Logger, optional + Pipeline logger + + Raises + ------ + ValueError + If the check images and the image are not on one pixel grid + + """ + with fits.open(cat_path) as hdul: + hdus = [hdu.copy() for hdu in hdul] + objects = next(h for h in hdus if h.name == "LDAC_OBJECTS") + data = objects.data + image = fits.getdata(image_path) + background = fits.getdata(background_path) + seg = fits.getdata(seg_path) + if not (seg.shape == image.shape == background.shape): + raise ValueError( + f"Segmentation {seg.shape}, image {image.shape} and background" + + f" {background.shape} must share one grid." + ) + vignets = data["VIGNET"] + col, row, n_tie = vignet_centres( + data["X_IMAGE"], data["Y_IMAGE"], vignets, image, background + ) + del image, background + seg_vignets = cut_stamps(seg, col, row, vignets.shape[1], 0) + del seg + new = fits.BinTableHDU.from_columns( + objects.columns.columns + [seg_vignet_column(seg_vignets)], + header=objects.header, + name="LDAC_OBJECTS", + ) + fits.HDUList( + [new if h is objects else h for h in hdus] + ).writeto(cat_path, overwrite=True) + if w_log: + centre = vignets.shape[1] // 2 + own = np.mean(seg_vignets[:, centre, centre] == data["NUMBER"]) + w_log.info( + f"SEG_VIGNET cut from {seg_path} for {len(col)} objects" + + f" ({n_tie} on a half pixel, resolved against VIGNET);" + + f" centre label = NUMBER for {own:.4f}" + ) + + def get_header_value(image_path, key): """Get Header Value. @@ -583,6 +702,7 @@ def get_check_image(self, check_image): if (len(check_image) == 1) & (check_image[0] == ""): check_type = ["NONE"] check_name = ["none"] + self.check_paths = {} else: check_type = [] check_name = [] @@ -596,6 +716,7 @@ def get_check_image(self, check_image): + self._num_str + ".fits" ) + self.check_paths = dict(zip(check_type, check_name)) self._cmd_line_extra += ( f' -CHECKIMAGE_TYPE {",".join(check_type)} ' diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index e45885b06..1f5beaa11 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -109,6 +109,27 @@ def sextractor_runner( # Parse SExtractor errors stdout, stderr = ss_inst.parse_errors(stderr, stdout) + # SEG_VIGNET (optional, environment-expanded boolean): add the + # SEGMENTATION check image's stamps, on each VIGNET's grid, as the + # SEG_VIGNET column ngmix's UberSeg blend handling reads. Needs the + # SEGMENTATION and BACKGROUND check images. + if config.has_option( + module_config_sec, "SEG_VIGNET" + ) and config.getexpandedboolean(module_config_sec, "SEG_VIGNET"): + missing = {"SEGMENTATION", "BACKGROUND"} - set(ss_inst.check_paths) + if missing: + raise ValueError( + f"SEG_VIGNET needs the {', '.join(sorted(missing))} check" + + " image(s) in CHECKIMAGE." + ) + ss.add_seg_vignet( + ss_inst.path_output_file, + ss_inst.check_paths["SEGMENTATION"], + input_file_list[0], + ss_inst.check_paths["BACKGROUND"], + w_log=w_log, + ) + # Run sextractor post processing if config.getboolean(module_config_sec, "MAKE_POST_PROCESS"): pos_params = config.getlist(module_config_sec, "WORLD_POSITION") diff --git a/tests/module/test_sextractor_seg_vignet.py b/tests/module/test_sextractor_seg_vignet.py new file mode 100644 index 000000000..55ef05ea2 --- /dev/null +++ b/tests/module/test_sextractor_seg_vignet.py @@ -0,0 +1,140 @@ +"""UNIT TESTS FOR SEXTRACTOR MODE'S SEG_VIGNET. + +``sextractor_script.add_seg_vignet`` cuts the SEGMENTATION check image on the +grid SExtractor cut each object's VIGNET on, and adds it to the sexcat as the +int32 ``SEG_VIGNET`` column. SExtractor centres VIGNET on the pixel nearest +its double-precision barycentre; the catalogue stores X_IMAGE / Y_IMAGE as +float32, which ties a few positions to an exact half pixel. Those objects +take whichever neighbouring centre reproduces their VIGNET from the image +minus the BACKGROUND check image, bit for bit. +""" + +import numpy as np +import numpy.testing as npt +import pytest +from astropy.io import fits + +from shapepipe.modules.sextractor_package import sextractor_script as ss + +NX, NY = 60, 50 +STAMP = 7 +BIG = np.float32(-1e30) + +# (NUMBER, x, y): 1-based double-precision barycentres. Object 3's x and +# object 4's y sit a float32 ulp either side of a half pixel, so the float32 +# catalogue shows an exact .5 for both but SExtractor rounded them apart. +OBJECTS = [ + (1, 20.2, 15.7), + (2, 1.3, 48.9), + (3, 30.5 + 5e-7, 30.1), + (4, 44.8, 22.5 - 5e-7), + (5, 59.9, 1.2), +] + + +def _scene(tmp_path): + rng = np.random.default_rng(3) + image = rng.normal(100, 5, (NY, NX)).astype(np.float32) + background = np.linspace(99, 101, NX * NY, dtype=np.float32).reshape( + NY, NX + ) + seg = np.zeros((NY, NX), np.int32) + for number, x, y in OBJECTS: + col, row = int(np.rint(x - 1)), int(np.rint(y - 1)) + seg[max(row - 2, 0):row + 3, max(col - 2, 0):col + 3] = number + number = np.array([o[0] for o in OBJECTS]) + x = np.array([o[1] for o in OBJECTS]) + y = np.array([o[2] for o in OBJECTS]) + col, row = np.rint(x - 1).astype(int), np.rint(y - 1).astype(int) + + # VIGNET as SExtractor writes it: image minus background, -1e30 off the + # image and on some neighbours' pixels. + vignets = ss.cut_stamps(image - background, col, row, STAMP, BIG) + seg_true = ss.cut_stamps(seg, col, row, STAMP, 0) + vignets[(seg_true != 0) & (seg_true != number[:, None, None])] = BIG + + paths = {name: str(tmp_path / f"{name}-001-001.fits") + for name in ("image", "background", "segmentation", "sexcat")} + fits.PrimaryHDU(image).writeto(paths["image"]) + fits.PrimaryHDU(background).writeto(paths["background"]) + fits.PrimaryHDU(seg).writeto(paths["segmentation"]) + objects = fits.BinTableHDU.from_columns([ + fits.Column(name="NUMBER", format="J", array=number), + fits.Column(name="X_IMAGE", format="E", array=x.astype(np.float32)), + fits.Column(name="Y_IMAGE", format="E", array=y.astype(np.float32)), + fits.Column(name="VIGNET", format=f"{STAMP * STAMP}E", + array=vignets.reshape(len(number), -1), + dim=f"({STAMP},{STAMP})"), + ], name="LDAC_OBJECTS") + imhead = fits.BinTableHDU.from_columns( + [fits.Column(name="Field Header Card", format="80A", + array=np.array(["HISTORY x"]))], name="LDAC_IMHEAD") + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto( + paths["sexcat"]) + return paths, seg, seg_true + + +def _add(paths): + ss.add_seg_vignet(paths["sexcat"], paths["segmentation"], paths["image"], + paths["background"]) + with fits.open(paths["sexcat"]) as hdul: + return [h.name for h in hdul], hdul["LDAC_OBJECTS"].data.copy() + + +def test_float32_catalogue_positions_tie_on_half_pixels(): + """The premise: the float32 columns lose which way objects 3 and 4 + rounded, so no rounding rule on the catalogue alone recovers both.""" + x3 = np.float32(OBJECTS[2][1]) + y4 = np.float32(OBJECTS[3][2]) + assert x3 % 1 == 0.5 and y4 % 1 == 0.5 + assert np.rint(OBJECTS[2][1] - 1) == np.floor(x3 - 1) + 1 + assert np.rint(OBJECTS[3][2] - 1) == np.floor(y4 - 1) + + +def test_seg_vignet_is_registered_with_vignet(tmp_path): + """SEG_VIGNET is the check image on VIGNET's grid for every object, the + half-pixel ties included: int32, VIGNET's shape, 0 off the image.""" + paths, _, seg_true = _scene(tmp_path) + names, data = _add(paths) + assert names == ["PRIMARY", "LDAC_IMHEAD", "LDAC_OBJECTS"] + seg_vignets = data["SEG_VIGNET"] + assert seg_vignets.dtype.kind == "i" and seg_vignets.dtype.itemsize == 4 + assert seg_vignets.shape == data["VIGNET"].shape + npt.assert_array_equal(seg_vignets, seg_true) + centre = STAMP // 2 + npt.assert_array_equal(seg_vignets[:, centre, centre], data["NUMBER"]) + + +def test_seg_vignet_leaves_the_other_columns_alone(tmp_path): + paths, _, _ = _scene(tmp_path) + with fits.open(paths["sexcat"]) as hdul: + before = hdul["LDAC_OBJECTS"].data.copy() + imhead = hdul["LDAC_IMHEAD"].data.copy() + _, after = _add(paths) + assert after.names == before.names + ["SEG_VIGNET"] + for name in before.names: + npt.assert_array_equal(after[name], before[name]) + with fits.open(paths["sexcat"]) as hdul: + npt.assert_array_equal(hdul["LDAC_IMHEAD"].data, imhead) + + +def test_sextractor_caller_names_its_check_images(tmp_path): + """The runner finds the check images add_seg_vignet reads by type.""" + caller = ss.SExtractorCaller( + [str(tmp_path / "image-001-001.fits"), + str(tmp_path / "weight-001-001.fits")], + str(tmp_path), "-001-001", "d.sex", "d.param", "d.conv", + True, False, False, False, False, False, False, + check_image=["BACKGROUND", "SEGMENTATION"], + ) + assert caller.check_paths == { + "BACKGROUND": f"{tmp_path}/background-001-001.fits", + "SEGMENTATION": f"{tmp_path}/segmentation-001-001.fits", + } + + +def test_seg_vignet_refuses_a_map_on_another_grid(tmp_path): + paths, seg, _ = _scene(tmp_path) + fits.PrimaryHDU(seg[:-1]).writeto(paths["segmentation"], overwrite=True) + with pytest.raises(ValueError, match="grid"): + _add(paths) From a4d6880cea5d77028b5f5b7871df2ffa4e357a46 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 00:44:26 +0200 Subject: [PATCH 18/44] ngmix: read UberSeg's seg stamps from the tile catalogue's SEG_VIGNET Tile_cat takes the stamps from the catalogue it already reads, and BLEND_HANDLING = uberseg fails before the object loop when the column is missing. SEG_VIGNET_PATH, its separate catalogue and its checks go. The stamp columns stay views into the loaded table, so it is held once, and BLEND_HANDLING is environment-expanded. Co-Authored-By: Claude Opus 5.5 --- src/shapepipe/modules/ngmix_package/ngmix.py | 93 ++++------- src/shapepipe/modules/ngmix_runner.py | 21 +-- tests/module/test_ngmix.py | 1 - tests/module/test_ngmix_uberseg.py | 156 ++++++++++++------- 4 files changed, 134 insertions(+), 137 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 471e62b2c..e51ca9a3d 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -539,22 +539,17 @@ class Tile_cat(): Parameters ---------- cat_path : str - Path to the tile SExtractor catalogue. - seg_cat_path : str, optional - Path to the coadd-frame segmentation VIGNET catalogue (a CLASSIC-mode - vignetmaker output cut from the tile ``SEGMENTATION`` check image), - row-aligned to ``cat_path``. When given, ``self.seg`` holds one integer - seg stamp per object for the ``"uberseg"`` blend handling; ``None`` - leaves ``self.seg`` unset, which only ``"uberseg"`` needs. + Path to the tile SExtractor catalogue. Its optional ``SEG_VIGNET`` + column, one integer coadd segmentation stamp per object on the grid + of its ``VIGNET``, becomes ``self.seg`` for the ``"uberseg"`` blend + handling; without it ``self.seg`` is ``None``. """ def __init__( self, cat_path, - seg_cat_path=None, ): self.cat_path = cat_path - self.seg_cat_path = seg_cat_path if cat_path: self.get_data(cat_path) @@ -571,44 +566,20 @@ def get_data(self, cat_path): self.ra = np.copy(data['XWIN_WORLD']) self.dec = np.copy(data['YWIN_WORLD']) - # Optional columns — may be absent in external (non-SExtractor) catalogs + # Optional columns — may be absent in external (non-SExtractor) catalogs. + # The stamp columns are the bulk of the table and are views into it, + # not copies, so it is held once (prepare_postage_stamps copies each + # object's stamp before changing it). self.flux = np.copy(data['FLUX_AUTO']) if 'FLUX_AUTO' in cols else None - self.vign = np.copy(data['VIGNET']) if 'VIGNET' in cols else None + self.vign = data['VIGNET'] if 'VIGNET' in cols else None - tile_cat.close() + # Coadd-frame segmentation stamp (integer labels, the catalogue's + # NUMBER), one per object on the grid of its VIGNET, overlaid + # unchanged on every epoch for uberseg neighbour masking + # (shapepipe#776). + self.seg = data['SEG_VIGNET'] if 'SEG_VIGNET' in cols else None - # Coadd-frame SExtractor segmentation stamp (integer labels), one per - # object and row-aligned to the tile catalogue, overlaid unchanged on - # every epoch for uberseg neighbour masking (shapepipe#776). None -> - # uberseg unavailable; ``"noisefill"`` does not read it. - self.seg = None - if self.seg_cat_path: - seg_cat = file_io.FITSCatalogue( - self.seg_cat_path, - SEx_catalogue=True, - ) - seg_cat.open() - seg_data = seg_cat.get_data() - # The seg VIGNETs are indexed by tile-catalogue row (self.seg[i]), - # so the two catalogues MUST be row-aligned. Fail loud at load if - # they are not — a silent length/order mismatch would hand every - # object the wrong footprint and quietly corrupt every mask. - if len(seg_data) != len(self.obj_id): - raise ValueError( - f"SEG_VIGNET_PATH '{self.seg_cat_path}' has" - + f" {len(seg_data)} rows but the tile catalogue has" - + f" {len(self.obj_id)}; the segmentation vignets must be" - + " row-aligned to the tile catalogue." - ) - if 'NUMBER' in seg_data.dtype.names: - if not np.array_equal(seg_data['NUMBER'], self.obj_id): - raise ValueError( - f"SEG_VIGNET_PATH '{self.seg_cat_path}' NUMBER column" - + " does not match the tile catalogue NUMBER; the" - + " segmentation vignets are misaligned or reordered." - ) - self.seg = np.copy(seg_data['VIGNET']) - seg_cat.close() + tile_cat.close() class Postage_stamp(): """Galaxy Postage Stamp. @@ -792,13 +763,11 @@ class Ngmix(object): blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment. ``"noisefill"`` (default) zero-weights and noise-fills the pixels marked -1e30 in the tile VIGNET on other - detections' footprints; ``"uberseg"`` ignores those markers, + detections' footprints; ``"uberseg"`` ignores those markers and zeroes the weight of neighbour-side pixels from the coadd - segmentation map and requires ``seg_cat_path``. Defect pixels are - filled under both (see :func:`prepare_ngmix_weights`). - seg_cat_path : str, optional - Path to the coadd-frame segmentation VIGNET catalogue (see - :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. + segmentation stamps, the tile catalogue's ``SEG_VIGNET`` column (see + :class:`Tile_cat`), which it requires. Defect pixels are filled + under both (see :func:`prepare_ngmix_weights`). dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. @@ -829,8 +798,7 @@ class Ngmix(object): IndexError If the length of the input file list is incorrect ValueError - If ``blend_handling`` or ``defect_fill`` is unknown, or ``"uberseg"`` - is selected without ``seg_cat_path``. + If ``blend_handling`` or ``defect_fill`` is unknown. """ @@ -849,7 +817,6 @@ def __init__( bkg_sub=True, centroid_source="wcs", blend_handling="noisefill", - seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, @@ -880,14 +847,6 @@ def __init__( + f" {DEFECT_FILLS}" ) - # Fail fast at construction (not deep in the per-epoch loop) when - # uberseg is requested without its segmentation input (shapepipe#776). - if blend_handling == "uberseg" and seg_cat_path is None: - raise ValueError( - "blend_handling='uberseg' requires SEG_VIGNET_PATH (the coadd" - + " SExtractor segmentation vignets); none configured." - ) - self._tile_cat_path = input_file_list[0] if bkg_sub: bkg_path, psf_path, weight_path, flag_path = ( @@ -927,7 +886,6 @@ def __init__( self._bkg_sub = bkg_sub self._centroid_source = centroid_source self._blend_handling = blend_handling - self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour self._metacal_psf = metacal_psf self._epoch_central_defect_radius = epoch_central_defect_radius @@ -1336,9 +1294,18 @@ def process(self): @sc [decision:shape_measurement.fit_initialisation,decision:shape_measurement.ngmix_seed_mode] """ - tile_cat = Tile_cat(self._tile_cat_path, self._seg_cat_path) + tile_cat = Tile_cat(self._tile_cat_path) vignet_cat = self._vignet_cat + # Fail before the per-object loop, whose try/except would otherwise + # drop every object one by one (shapepipe#776). + if self._blend_handling == "uberseg" and tile_cat.seg is None: + raise ValueError( + "BLEND_HANDLING = uberseg needs the tile catalogue's" + + f" SEG_VIGNET column, which {self._tile_cat_path} lacks;" + + " write it at tile detection (SEG_VIGNET = True)." + ) + check_wcs_centroid_offset( self._centroid_source, tile_cat, vignet_cat.gal_vign_cat ) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 08113aa84..819291a4a 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -100,24 +100,6 @@ def ngmix_runner( else: input_file_list = input_file_list[:wcs_idx] - # SEG_VIGNET_PATH (optional): coadd-frame SExtractor segmentation vignets - # (a CLASSIC-mode vignetmaker output), row-aligned to the tile catalogue. - # Required for BLEND_HANDLING = uberseg; when set, the file must exist for - # every tile (missing file -> error). Read on Tile_cat, not via Vignet, so - # it is threaded to Ngmix as its own argument rather than into - # input_file_list. - if config.has_option(module_config_sec, "SEG_VIGNET_PATH"): - seg_vignet_path = config.getexpanded( - module_config_sec, - "SEG_VIGNET_PATH", - ).format(file_number_string=file_number_string) - if not os.path.exists(seg_vignet_path): - raise FileNotFoundError( - f"Segmentation vignet file not found: {seg_vignet_path}" - ) - else: - seg_vignet_path = None - # Batch save option if config.has_option(module_config_sec, "SAVE_BATCH"): save_batch = config.getint( @@ -152,7 +134,7 @@ def ngmix_runner( # zero-weight, invalid-RMS or off-tile) are zero-weighted and filled # under both (DEFECT_FILL below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): - blend_handling = config.get(module_config_sec, "BLEND_HANDLING") + blend_handling = config.getexpanded(module_config_sec, "BLEND_HANDLING") else: blend_handling = "noisefill" @@ -263,7 +245,6 @@ def ngmix_runner( bkg_sub=bkg_sub, centroid_source=centroid_source, blend_handling=blend_handling, - seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, metacal_psf=metacal_psf, epoch_central_defect_radius=epoch_central_defect_radius, diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index a69aa0e69..8d64a43e2 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -652,7 +652,6 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags ) inst = object.__new__(Ngmix) inst._tile_cat_path = "in-memory-tile" - inst._seg_cat_path = None inst._vignet_cat = SimpleNamespace( gal_vign_cat=galaxies, psf_vign_cat=galaxies, close=lambda: None, ) diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 5c20789f6..561601151 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -17,11 +17,13 @@ import numpy as np import numpy.testing as npt import pytest +from astropy.io import fits from scipy import ndimage from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package.ngmix import ( Ngmix, + Tile_cat, central_seg_label, prepare_ngmix_weights, seg_has_neighbour, @@ -350,60 +352,108 @@ def test_check_central_seg_label_missing_number_raises(tmp_path): ngmix._check_central_seg_label(seg, obj_id=99) -# --- FIX 3a: uberseg without seg_cat_path fails at construction ------------- - -def test_ngmix_init_uberseg_without_seg_cat_raises(tmp_path): - """blend_handling='uberseg' with seg_cat_path=None raises at __init__.""" +# --- the seg stamps ride the tile catalogue as SEG_VIGNET ------------------- + +def _write_tile_cat(path, seg_vignets=None): + """A three-object LDAC tile catalogue, with SEG_VIGNET when given.""" + n_obj, size = 3, 5 + cols = [ + fits.Column(name="NUMBER", format="J", array=np.array([4, 5, 9])), + fits.Column(name="XWIN_WORLD", format="D", array=np.zeros(n_obj)), + fits.Column(name="YWIN_WORLD", format="D", array=np.zeros(n_obj)), + fits.Column(name="VIGNET", format=f"{size * size}E", + array=np.ones((n_obj, size * size), np.float32), + dim=f"({size},{size})"), + ] + if seg_vignets is not None: + cols.append(fits.Column(name="SEG_VIGNET", format=f"{size * size}J", + array=seg_vignets.reshape(n_obj, -1), + dim=f"({size},{size})")) + imhead = fits.BinTableHDU.from_columns( + [fits.Column(name="Field Header Card", format="1A", array=["x"])], + name="LDAC_IMHEAD", + ) + fits.HDUList([ + fits.PrimaryHDU(), imhead, + fits.BinTableHDU.from_columns(cols, name="LDAC_OBJECTS"), + ]).writeto(path) + return path + + +def test_tile_cat_reads_seg_vignet(tmp_path): + """Tile_cat's seg stamps are the catalogue's SEG_VIGNET, row for row; + a catalogue without the column has none.""" + seg = np.arange(3 * 25, dtype=np.int32).reshape(3, 5, 5) + with_seg = Tile_cat(str(_write_tile_cat(tmp_path / "a.fits", seg))) + npt.assert_array_equal(with_seg.seg, seg) + assert with_seg.seg.dtype.kind == "i" + assert Tile_cat(str(_write_tile_cat(tmp_path / "b.fits"))).seg is None + + +def test_tile_cat_holds_the_stamp_columns_once(tmp_path): + """VIGNET and SEG_VIGNET are views into the one loaded table: copying + either would double the bulk of every ngmix chunk's catalogue memory.""" + seg = np.zeros((3, 5, 5), np.int32) + tile = Tile_cat(str(_write_tile_cat(tmp_path / "a.fits", seg))) + # Interleaved fields of one record buffer: their extents overlap. + assert np.may_share_memory(tile.vign, tile.seg) + + +def test_uberseg_without_seg_vignet_fails_loudly(tmp_path): + """BLEND_HANDLING = uberseg on a catalogue without SEG_VIGNET raises + before any object is fitted, rather than dropping every object.""" + cat = _write_tile_cat(tmp_path / "tile_cat.fits") names = ("gal", "bkg", "psf", "weight", "flag", "headers") paths = [tmp_path / f"{name}.sqlite" for name in names] for path in paths: SqliteDict(str(path)).close() - with pytest.raises(ValueError, match="requires SEG_VIGNET_PATH"): - Ngmix( - ["tile_cat.fits"] + [str(p) for p in paths[:5]], - str(tmp_path), "-001-001", 30.0, 0.186, str(paths[5]), - _RecordingLogger(), - blend_handling="uberseg", - seg_cat_path=None, - ) - - -# --- FIX 3b: runner raises when SEG_VIGNET_PATH is set but missing ---------- - -class _FakeConfig: - """Config stub: SEG_VIGNET_PATH is the only present option, and it resolves - to ``seg_path`` (a path the test leaves nonexistent).""" - - def __init__(self, seg_path): - self._seg_path = seg_path - - def getfloat(self, _sec, _key): - return 30.0 if _key == "MAG_ZP" else 0.186 - - def getboolean(self, _sec, _key, fallback=False): - return fallback - - def has_option(self, _sec, key): - return key == "SEG_VIGNET_PATH" - - def getexpanded(self, _sec, _key): - return self._seg_path - - -def test_runner_missing_seg_vignet_file_raises(tmp_path): - """SEG_VIGNET_PATH configured but the resolved file is absent -> the runner - fails fast with FileNotFoundError, before constructing Ngmix.""" - from shapepipe.modules.ngmix_runner import ngmix_runner - - input_file_list = ["tile_cat.fits"] + [f"in{i}.sqlite" for i in range(6)] - seg_path = str(tmp_path / "does_not_exist.fits") - with pytest.raises(FileNotFoundError, match="Segmentation vignet file"): - ngmix_runner( - input_file_list, - {"output": str(tmp_path)}, - "-001-001", - _FakeConfig(seg_path), - "NGMIX_RUNNER", - _RecordingLogger(), - ) - + ngmix = Ngmix( + [str(cat)] + [str(p) for p in paths[:5]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[5]), + _RecordingLogger(), blend_handling="uberseg", + ) + with pytest.raises(ValueError, match="SEG_VIGNET"): + ngmix.process() + + +def test_runner_reads_blend_handling_from_the_environment( + tmp_path, monkeypatch, +): + """The committed ini's ``${SP_BLEND_HANDLING:-noisefill}`` reaches Ngmix + as noisefill when the variable is unset and as its value when set.""" + from shapepipe.modules import ngmix_runner as runner_module + from shapepipe.pipeline.config import CustomParser + + names = ("cat", "gal", "bkg", "psf", "weight", "flag", "headers") + paths = [str(tmp_path / f"{name}.sqlite") for name in names] + for path in paths[1:]: + SqliteDict(path).close() + config = CustomParser() + config.add_section("NGMIX_RUNNER") + for key, value in {"MAG_ZP": "30.0", "ID_OBJ_MIN": "-1", + "ID_OBJ_MAX": "-1", + "BLEND_HANDLING": "${SP_BLEND_HANDLING:-noisefill}", + }.items(): + config.set("NGMIX_RUNNER", key, value) + + seen = [] + + class _Stop(Exception): + pass + + def fake_ngmix(*_args, **kwargs): + seen.append(kwargs["blend_handling"]) + raise _Stop + + monkeypatch.setattr(runner_module, "Ngmix", fake_ngmix) + runner = getattr(runner_module.ngmix_runner, "__wrapped__", + runner_module.ngmix_runner) + for env in (None, "uberseg"): + if env is None: + monkeypatch.delenv("SP_BLEND_HANDLING", raising=False) + else: + monkeypatch.setenv("SP_BLEND_HANDLING", env) + with pytest.raises(_Stop): + runner(paths, {"output": str(tmp_path)}, "-001-001", config, + "NGMIX_RUNNER", _RecordingLogger()) + assert seen == ["noisefill", "uberseg"] From 22cdd1971c25b2fb6db9ba65179a85ab1b905614 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 00:50:57 +0200 Subject: [PATCH 19/44] Carry blend_handling to tile_detect and ngmix as two prologue exports blend_handling: uberseg (config.yaml, default noisefill) exports SP_SEG_VIGNET=True to tile_detect and SP_BLEND_HANDLING=uberseg to tile_ngmix, and adds the SEG_VIGNET column's memory to each ngmix chunk. The committed inis read them as ${SP_SEG_VIGNET:-False} and ${SP_BLEND_HANDLING:-noisefill}, so noisefill plans the campaign it planned without the knob and the params pin does not move. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_workflow_tile_detection.py | 25 +++++ tests/workflow/README.md | 2 + tests/workflow/test_dag.py | 105 ++++++++++++++++++ workflow/README.md | 6 + workflow/Snakefile | 8 ++ workflow/config.yaml | 8 ++ .../config/cfis/config_tile_Ng_template.ini | 5 + workflow/config/cfis/config_tile_Sx.ini | 6 + workflow/config/cfis/config_tile_Uc.ini | 6 + workflow/rules/tile.smk | 53 ++++++++- workflow/scripts/run_config.py | 4 + 11 files changed, 223 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_workflow_tile_detection.py b/tests/unit/test_workflow_tile_detection.py index 24775304e..d42c6c667 100644 --- a/tests/unit/test_workflow_tile_detection.py +++ b/tests/unit/test_workflow_tile_detection.py @@ -176,3 +176,28 @@ def test_catalogue_retrieve_follows_the_prefix(source, retrieve): config = {"tile_detection": "unions_catalogue", "inputs": {"catalogues": source}} assert run_config.catalogue_source(config) == (source, retrieve) + + +# --- blend handling -------------------------------------------------------- + + +def test_blend_handlings_mirror_the_ngmix_module(): + """run_config.BLEND_HANDLINGS is a copy; the copy must stay true. + + The Snakefile validates `blend_handling:` against it in the launcher venv, + outside the container where shapepipe is importable, so the tuple is + mirrored rather than imported, and read out of the source text here for + the same reason: this file is container-free. + """ + import re + + src = (REPO_ROOT / "src" / "shapepipe" / "modules" / "ngmix_package" + / "ngmix.py").read_text() + match = re.search(r"^BLEND_HANDLINGS = \(([^)]*)\)", src, re.M) + assert match, "ngmix.py no longer defines BLEND_HANDLINGS" + assert _load("run_config").BLEND_HANDLINGS == tuple( + part.strip().strip('"\'') for part in match.group(1).split(",") + if part.strip()) + committed = _load("run_config").load( + str(REPO_ROOT / "workflow" / "config.yaml")) + assert committed["blend_handling"] == "noisefill" diff --git a/tests/workflow/README.md b/tests/workflow/README.md index c3576f08c..a6fbfeb04 100644 --- a/tests/workflow/README.md +++ b/tests/workflow/README.md @@ -76,6 +76,8 @@ Apply mutations only to a disposable checkout, run the named test without `--upd | `test_clean_exposure_waits_on_persist_iff_psf` | Drop the persist edge; make it unconditional under fake PSFs; drop vignets consumers; remove the in-scope consumer filter. | | `test_final_cat_merge_reads_every_ready_tile` | Drop one ready tile; append an out-of-scope tile. | | `test_products_use_products_dir_and_run_name` | Rename either merged catalogue or the persist manifest; route products to scratch; derive `CAMPAIGN` from the products directory's basename. | +| `test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg` | Export `BLEND_ENV` under noisefill; drop `blend_env(...)` from either tile_detect or tile_ngmix; export the wrong value; set `NGMIX_SEG_MEM_MB` to 0; give a committed ini's `SEG_VIGNET` / `BLEND_HANDLING` a literal or the wrong default. | +| `test_unknown_blend_handling_fails_during_parse` | Remove the Snakefile's `blend_handling` check. | | `test_missing_run_fails_during_parse` | Remove `run` from `run_config.REQUIRED`; literal paths must still receive the required-key diagnostic, not a later `KeyError`. | | `test_unit_pre_changes_at_campaign_boundary` | Append a line to `unit_pre`; change one rule's `params.pre`; change one shell; change a rendered thread count. | | `test_params_pin_ignores_fixture_root` | Remove fixture-root normalization. | diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index 46ae93fc0..a7b8358b6 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -119,3 +119,108 @@ def test_mccd_is_refused_during_parse(tmp_path, resolve_dag): with pytest.raises(WorkflowError, match=r"psf_model=mccd: PSF persistence"): with resolve_dag(campaign): pytest.fail("psf_model=mccd must be refused at parse time") + + +# --- blend_handling --------------------------------------------------------- + +BLEND_EXPORTS = {"tile_detect": {"SP_SEG_VIGNET": "True"}, + "tile_ngmix": {"SP_BLEND_HANDLING": "uberseg"}} +# The option each stage's committed ini reads the export through, and the +# value it must resolve to with and without it. +BLEND_OPTIONS = {"tile_detect": ("SEG_VIGNET", "False", "True"), + "tile_ngmix": ("BLEND_HANDLING", "noisefill", "uberseg")} + + +def _surface(campaign, resolve_dag): + """Every job's prologue, shell and memory, keyed by rule and wildcards.""" + with resolve_dag(campaign) as dag: + return dag.rule_names, { + (job.rule.name, tuple(sorted(job.wildcards_dict.items()))): ( + getattr(job.params, "pre", None), job.shellcmd, + job.resources.get("mem_mb")) + for job in dag.jobs + } + + +def _committed_value(shell, config_dir, option, env, monkeypatch): + """``option`` of the module section of the ini ``shell`` runs, expanded + under ``env`` as ShapePipe expands it.""" + from shapepipe.pipeline.config import CustomParser + + name = shell.split('shapepipe_run -c "$SP_CONFIG/')[1].split('"')[0] + parser = CustomParser() + parser.optionxform = str + assert parser.read(config_dir / name) + section = next(s for s in parser.sections() if s.endswith("_RUNNER")) + for key in ("SP_SEG_VIGNET", "SP_BLEND_HANDLING"): + monkeypatch.delenv(key, raising=False) + for key, value in env.items(): + monkeypatch.setenv(key, value) + return parser.getexpanded(section, option) + + +@pytest.mark.parametrize("detection", ["sextractor", "unions_catalogue"]) +def test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg( + tmp_path, resolve_dag, monkeypatch, detection): + """noisefill plans exactly the campaign without the knob; uberseg only + adds its two exports to tile_detect's and tile_ngmix's prologues and the + seg stamps' memory to tile_ngmix, and each export turns its committed + ini's option from the noise-fill default to the uberseg value.""" + surfaces = {} + for blend in (None, "noisefill", "uberseg"): + campaign = Campaign(tmp_path / str(blend), "data", "psfex") + campaign.config["tile_detection"] = detection + if blend is not None: + campaign.config["blend_handling"] = blend + campaign.write_config() + surfaces[blend] = _surface(campaign, resolve_dag) + + def normalized(blend): + rules, jobs = surfaces[blend] + root = str(tmp_path / str(blend)) + return rules, {key: tuple(v.replace(root, "") + if isinstance(v, str) else v for v in value) + for key, value in jobs.items()} + + assert normalized("noisefill") == normalized(None) + rules, noisefill = normalized("noisefill") + uberseg_rules, uberseg = normalized("uberseg") + assert uberseg_rules == rules + assert uberseg.keys() == noisefill.keys() + config_dir = Path(__file__).parents[2] / "workflow" / "config" / "cfis" + for key, (pre, shell, mem) in uberseg.items(): + rule = key[0] + nf_pre, nf_shell, nf_mem = noisefill[key] + exports = BLEND_EXPORTS.get(rule, {}) + lines = {f"export {name}='{value}'" for name, value in exports.items()} + + def without_exports(text): + if text is None: + return None + return "\n".join(line for line in text.split("\n") + if line not in lines) + + # The rendered shell carries the prologue, so it moves with it and + # nowhere else. + assert without_exports(shell) == nf_shell, key + if pre is None: + assert nf_pre is None and not exports, key + continue + assert lines <= set(pre.split("\n")), key + assert without_exports(pre) == nf_pre, key + if exports: + option, default, value = BLEND_OPTIONS[rule] + assert _committed_value(shell, config_dir, option, {}, + monkeypatch) == default + assert _committed_value(shell, config_dir, option, exports, + monkeypatch) == value + assert mem == (nf_mem + 500 if rule == "tile_ngmix" else nf_mem), key + + +def test_unknown_blend_handling_fails_during_parse(tmp_path, resolve_dag): + campaign = Campaign(tmp_path / "campaign", "data", "psfex") + campaign.config["blend_handling"] = "mof" + campaign.write_config() + with pytest.raises(WorkflowError, match=r"Invalid blend_handling='mof'"): + with resolve_dag(campaign): + pytest.fail("an unknown blend_handling must fail at parse time") diff --git a/workflow/README.md b/workflow/README.md index 6e91b344f..26f00583c 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -34,6 +34,12 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' # tile is detected with SExtractor; the default for image sims). Either way # make_cat writes TILE_UNIQUE_ID = tile_id * 10**6 + NUMBER and ngmix masks # the same neighbours. +# `blend_handling` is ngmix's neighbour treatment: `noisefill` (the default) +# or `uberseg`, under which tile_detect adds the coadd segmentation stamps to +# the sexcat as SEG_VIGNET and ngmix masks neighbours from them. It is chosen +# per campaign: flipping it on an existing run dir reruns tile_detect and the +# shape chain for every finished tile, and reclaimed exposures make that +# destructive. An uberseg comparison needs its own `run:`. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. diff --git a/workflow/Snakefile b/workflow/Snakefile index c9124fcd2..85b6af482 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -188,6 +188,14 @@ try: except ValueError as err: raise WorkflowError(str(err)) from None +# How ngmix treats a stamp's neighbours; tile.smk's blend handling section +# carries it to tile_detect and tile_ngmix. +BLEND_HANDLING = config.get("blend_handling", "noisefill") +if BLEND_HANDLING not in run_config.BLEND_HANDLINGS: + raise WorkflowError( + f"Invalid blend_handling={BLEND_HANDLING!r}; expected one of " + f"{sorted(run_config.BLEND_HANDLINGS)}.") + # SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two # invocations of `sp run`, `passthrough` on direct commands (`sp --unlock`, `sp # --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index diff --git a/workflow/config.yaml b/workflow/config.yaml index b6f22a01e..913245b87 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -20,6 +20,14 @@ input_type: data # campaign's merged catalogues. Required, and set by the run config. # run: smk-g6 +# How ngmix treats a stamp's neighbours (its BLEND_HANDLING): `noisefill` +# noise-fills them; `uberseg` zeroes their weight from the coadd segmentation +# map, whose stamps tile_detect then writes into the sexcat as SEG_VIGNET. +# Chosen per campaign: flipping it on an existing run dir reruns tile_detect +# and everything after it for every finished tile, and reclaimed exposures +# make that destructive -- an uberseg comparison needs its own `run:`. +blend_handling: noisefill + # Default settings per input_type, can be overridden by a user-defined run config file. # - tile_detection: allowed are `unions_catalogue` (fetch official UNIONS catalogue # from vos; recommended for data); `sextractor` (detect objects via a SExtractor diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index 61addb4f2..dd2119b5f 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -125,3 +125,8 @@ PIXEL_SCALE = 0.186 # via ShapePipe's getexpanded (env-expanded, not plain getint). ID_OBJ_MIN = $NGMIX_ROW_MIN ID_OBJ_MAX = $NGMIX_ROW_MAX + +# Neighbour treatment: noisefill, or uberseg, which reads the tile catalogue's +# SEG_VIGNET column. The workflow sets SP_BLEND_HANDLING under +# blend_handling: uberseg only (workflow/config.yaml). +BLEND_HANDLING = ${SP_BLEND_HANDLING:-noisefill} diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 0c05c38b5..4dcef22a6 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -111,6 +111,12 @@ BKG_FROM_HEADER = False # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES CHECKIMAGE = BACKGROUND, SEGMENTATION +# Write the SEGMENTATION check image's stamps, cut on each VIGNET's grid, as +# the int32 SEG_VIGNET column ngmix's UberSeg blend handling reads (needs the +# BACKGROUND and SEGMENTATION check images). The workflow sets SP_SEG_VIGNET +# under blend_handling: uberseg only (workflow/rules/tile.smk). +SEG_VIGNET = ${SP_SEG_VIGNET:-False} + # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini index ab9d04742..0f1a98cd5 100644 --- a/workflow/config/cfis/config_tile_Uc.ini +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -82,6 +82,12 @@ VIGNET_SIZE = 51 # @sc [decision:detection.catalogue_neighbour_marking] SEGMENTATION = True +# Write that map's stamps, relabelled to the catalogue's NUMBER and cut on each +# VIGNET's grid, as the int32 SEG_VIGNET column ngmix's UberSeg blend handling +# reads. The workflow sets +# SP_SEG_VIGNET under blend_handling: uberseg only (workflow/rules/tile.smk). +SEG_VIGNET = ${SP_SEG_VIGNET:-False} + ## Post-processing # Necessary for tiles, to enable multi-exposure processing diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index f105d23b4..96f2b7aa7 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -54,6 +54,16 @@ mode. The converter writes run_sp_tile_Sx/read_ext_sexcat_runner, and the rule l sextractor_runner, the one path every downstream config reads; the manifest is tile_detect.json in both modes, so tile_vignets onwards is the same DAG. +``blend_handling: uberseg`` switches ngmix from noise-filling neighbours to +UberSeg's hard mask, which reads one coadd segmentation stamp per object from +the sexcat's SEG_VIGNET column, cut on the grid of the object's VIGNET. +tile_detect writes that column in both modes: the converter cuts it from the +catalogue's segmentation map relabelled to its NUMBER (-1 on footprints no +object claims, 0 on sky), SExtractor mode from the SEGMENTATION check image, +labelled by the same run's NUMBER. The switch is two prologue variables and +nothing else (blend_env, below), so a noise-fill campaign runs exactly the +chain it ran without the knob. + There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe generates no masks: tiles have no instrument flag image of their own, so tile_detect runs SExtractor with FLAG_IMAGE = False against @@ -472,6 +482,25 @@ rule tile_merge_headers: shell: sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") +# --- blend handling ------------------------------------------------------- + +# What blend_handling: uberseg exports into a stage's prologue; nothing under +# noisefill, so noise-fill prologues (rerun triggers) are those without the +# knob. The committed inis read each variable with its noise-fill default: +# SEG_VIGNET = ${SP_SEG_VIGNET:-False} in config_tile_Sx.ini and +# config_tile_Uc.ini, BLEND_HANDLING = ${SP_BLEND_HANDLING:-noisefill} in +# config_tile_Ng_template.ini. Being in the prologue, a flip reruns +# tile_detect, and with it the tile's whole shape chain. +BLEND_ENV = { + "tile_detect": {"SP_SEG_VIGNET": "True"}, + "tile_ngmix": {"SP_BLEND_HANDLING": "uberseg"}, +} if BLEND_HANDLING == "uberseg" else {} + + +def blend_env(stage): + return BLEND_ENV.get(stage, {}) + + # The tile's galaxy sample. One of two definitions of tile_detect, chosen at # parse time by the run config; both produce manifests/tile_detect.json and # run_sp_tile_Sx/sextractor_runner/output/sexcat.fits. @@ -487,7 +516,8 @@ if TILE_DETECTION == "sextractor": log: f"{TILE_DIR}/logs/tile_detect.json" params: - pre = lambda wc: unit_pre("tile_detect", wc.tile), + pre = lambda wc: unit_pre("tile_detect", wc.tile, + env=blend_env("tile_detect")), script_hash = SCRIPT_HASH threads: 8 resources: @@ -540,11 +570,13 @@ else: params: pre = lambda wc: unit_pre( "tile_detect", wc.tile, - env={"SP_TILE_DETECTION": TILE_DETECTION}), + env={"SP_TILE_DETECTION": TILE_DETECTION, + **blend_env("tile_detect")}), script_hash = SCRIPT_HASH threads: 1 resources: - # The whole tile image plus one 51x51 float32 stamp per object. + # The whole tile image plus one 51x51 float32 stamp per object + # (and one int32 SEG_VIGNET stamp under uberseg). mem_mb = lambda wc, attempt: 8000 * attempt, runtime = 60 shell: @@ -610,6 +642,13 @@ rule tile_vignets: sp_shell("tile_vignets", f"config_tile_PiViVi_{PSF_MODEL}.ini", check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') +# Each ngmix chunk's extra memory under uberseg: Tile_cat holding the tile's +# SEG_VIGNET column, a view into the loaded table like VIGNET. Measured on DR6 +# 202.301 (36065 rows, 51x51): Tile_cat peak and resident RSS 955 MiB with the +# column, 595 without (+360 MiB, the column's size). 500 covers 186.307's +# ~37.9k rows (~375 MiB) with a third to spare. +NGMIX_SEG_MEM_MB = 500 if BLEND_HANDLING == "uberseg" else 0 + # ngmix shape measurement — N chunks per tile (D4). Each chunk LOOKS UP its own # CLOSED catalogue-row range in the file tile_vignets materialised at the top of this # group job (TILE_NGMIX_RANGES); the ranges are knowable only at EXECUTION time, @@ -640,7 +679,8 @@ rule tile_ngmix: f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" params: pre = lambda wc: unit_pre("tile_ngmix", wc.tile, - env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS, + **blend_env("tile_ngmix")}, # Two steps, not `eval "$(...)"`: a command substitution inside eval # discards the script's exit status, so a missing sexcat would fall # through to shapepipe_run with an unset range and fail as something @@ -799,7 +839,10 @@ rule tile_ngmix: # margin over the worst tile measured, and the first thing to give would # be the page cache holding the store. Take it only with a measurement # of cache behaviour under pressure, not on the arithmetic alone. - mem_mb = lambda wc, attempt: 5000 * attempt, + # + # Under uberseg each chunk also holds the tile's SEG_VIGNET column, + # int32 and the size of VIGNET: NGMIX_SEG_MEM_MB below. + mem_mb = lambda wc, attempt: (5000 + NGMIX_SEG_MEM_MB) * attempt, # 120 on the FIRST attempt, and ATTEMPT-SCALED after it. MEASURED # two ways, and the second is why the margin is thinner than it looks: # * alone (job 20795277, tile 198.305 chunk 1): ~76 min for 3547 diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py index 669a3ec60..7cf9c18c3 100644 --- a/workflow/scripts/run_config.py +++ b/workflow/scripts/run_config.py @@ -28,6 +28,10 @@ # never mention `$run` must still set it. REQUIRED = ("run", "tile_list", "inputs.tiles", "inputs.exposures", "outputs.run_dir", "outputs.index_db") +# ngmix's BLEND_HANDLING values, which `blend_handling` takes. A copy, because +# the Snakefile validates it outside the container; tests/unit holds it equal +# to ngmix_package.ngmix.BLEND_HANDLINGS. +BLEND_HANDLINGS = ("noisefill", "uberseg") def merge(base, over): From 0eb2cfb1a00174de15fce2fc76f9a6476323e660 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 00:50:57 +0200 Subject: [PATCH 20/44] make_cat: leave SEG_VIGNET out of final_cat, with VIGNET Co-Authored-By: Claude Opus 5.5 --- src/shapepipe/modules/make_cat_package/make_cat.py | 4 +++- tests/module/test_read_ext_sexcat.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index b417f2842..8a7a8f617 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -113,7 +113,8 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): sexcat_path : str Path to SExtractor catalogue to save remove_vignet : bool - If ``True`` will not save the ``VIGNET`` field into the final catalogue + If ``True`` will not save the ``VIGNET`` and ``SEG_VIGNET`` stamp + fields into the final catalogue Returns ------- @@ -127,6 +128,7 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): data = np.copy(sexcat_file.get_data()) if remove_vignet: data = remove_field_name(data, "VIGNET") + data = remove_field_name(data, "SEG_VIGNET") cat_size = len(data) tile_name = os.path.basename(sexcat_path) diff --git a/tests/module/test_read_ext_sexcat.py b/tests/module/test_read_ext_sexcat.py index 44dee1a01..0a2f6cfed 100644 --- a/tests/module/test_read_ext_sexcat.py +++ b/tests/module/test_read_ext_sexcat.py @@ -457,6 +457,18 @@ def test_seg_vignet_needs_the_segmentation_map(tmp_path): stamp_size=STAMP, seg_vignet=True) +def test_seg_vignet_stays_out_of_the_final_catalogue(tmp_path): + """make_cat drops SEG_VIGNET with VIGNET: the stamps are ngmix inputs, + not catalogue columns.""" + _convert_with_seg(tmp_path, seg_vignet=True) + final = make_cat.prepare_final_cat_file(str(tmp_path), "-301-279") + make_cat.save_sextractor_data(final, str(tmp_path / "sexcat-301-279.fits")) + with fits.open(tmp_path / "final_cat-301-279.fits") as hdul: + names = hdul["RESULTS"].data.names + assert "VIGNET" not in names and "SEG_VIGNET" not in names + assert "NUMBER" in names + + # --- the runner's output against the completeness table ------------------- From f4e9515a40e6c531c9ee0494c6a160b7055378b9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 01:01:30 +0200 Subject: [PATCH 21/44] astra.yaml: UberSeg reads the tile catalogue's SEG_VIGNET Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astra.yaml b/astra.yaml index bf019aa51..b3e5c827d 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1501,8 +1501,8 @@ analyses: uberseg: label: UberSeg, weight-only description: >- - As in DES Y1/Y3. Needs the coadd segmentation stamp - (SEG_VIGNET_PATH). DILATE_NEIGHBOUR absorbs the coadd-vs-epoch + As in DES Y1/Y3. Needs the coadd segmentation stamp (the tile + catalogue's SEG_VIGNET). DILATE_NEIGHBOUR absorbs the coadd-vs-epoch overlay offset: ShapePipe reuses one coadd seg stamp for every epoch where MEDS reprojects it. insights: [mask_uberseg_weight_only, mask_uberseg_neighbour_bias, mask_des_y1_uberseg_only, mask_blend_bias_detection] From d55abc840561fb3e1115380505072f7ae95d9d20 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 01:21:20 +0200 Subject: [PATCH 22/44] Drop inherited blend variables before any rule A noisefill campaign launched from a shell that exports SP_SEG_VIGNET or SP_BLEND_HANDLING (bare, or in the APPTAINERENV_/SINGULARITYENV_ forms that survive --cleanenv) would hand them to every job through the slurm executor's --export=ALL. The Snakefile now pops them at parse time, so only blend_env's exports under uberseg set them. Prologues and shells are unchanged; the DAG harness gains a launch_env to test it. Co-Authored-By: Claude Opus 5.5 --- tests/workflow/conftest.py | 2 +- tests/workflow/harness.py | 10 ++++++++-- tests/workflow/test_dag.py | 41 ++++++++++++++++++++++++++++++++++++++ workflow/Snakefile | 10 ++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/tests/workflow/conftest.py b/tests/workflow/conftest.py index 134a9c62a..f08beaaa8 100644 --- a/tests/workflow/conftest.py +++ b/tests/workflow/conftest.py @@ -22,7 +22,7 @@ def campaign(request, tmp_path): @pytest.fixture def resolve_dag(monkeypatch): """Expose the resolver so tests can also assert parse-time failures.""" - return lambda campaign: resolve(campaign, monkeypatch) + return lambda campaign, **kwargs: resolve(campaign, monkeypatch, **kwargs) @pytest.fixture diff --git a/tests/workflow/harness.py b/tests/workflow/harness.py index 1c8420f26..abe0783ad 100644 --- a/tests/workflow/harness.py +++ b/tests/workflow/harness.py @@ -206,8 +206,12 @@ def load_profile(name): @contextmanager -def resolve(campaign, monkeypatch): - """Resolve ``all`` in an isolated state directory, without running jobs.""" +def resolve(campaign, monkeypatch, launch_env=None): + """Resolve ``all`` in an isolated state directory, without running jobs. + + ``launch_env`` adds variables to the scrubbed environment, as if the + launching shell had exported them. + """ from snakemake import workflow as sm_workflow scripts = REPO / "workflow" / "scripts" @@ -234,6 +238,8 @@ def resolve(campaign, monkeypatch): "XDG_CACHE_HOME": campaign.root / "cache", }.items(): patch.setenv(name, str(value)) + for name, value in (launch_env or {}).items(): + patch.setenv(name, str(value)) profile = load_profile("candide") try: with SnakemakeApi() as api: diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index a7b8358b6..67d0282e1 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -1,5 +1,6 @@ """Resolved-job checks for campaign scope, product paths, and PSF custody.""" +import os from collections import Counter from pathlib import Path @@ -224,3 +225,43 @@ def test_unknown_blend_handling_fails_during_parse(tmp_path, resolve_dag): with pytest.raises(WorkflowError, match=r"Invalid blend_handling='mof'"): with resolve_dag(campaign): pytest.fail("an unknown blend_handling must fail at parse time") + + +INHERITED_BLEND_ENV = { + f"{prefix}{name}": value + for prefix in ("", "APPTAINERENV_", "SINGULARITYENV_") + for name, value in (("SP_SEG_VIGNET", "True"), + ("SP_BLEND_HANDLING", "uberseg")) +} + + +def test_noisefill_ignores_blend_variables_in_the_launch_shell( + tmp_path, resolve_dag): + """A noisefill campaign launched from a shell that still exports the + uberseg variables plans exactly the campaign launched from a clean one, + and the parse leaves none of them in the environment jobs inherit (the + slurm executor submits with --export=ALL from this process).""" + clean = Campaign(tmp_path / "clean", "data", "psfex") + _, clean_jobs = _surface(clean, resolve_dag) + + dirty = Campaign(tmp_path / "dirty", "data", "psfex") + with resolve_dag(dirty, launch_env=INHERITED_BLEND_ENV) as dag: + leaked = sorted(set(INHERITED_BLEND_ENV) & set(os.environ)) + dirty_jobs = { + (job.rule.name, tuple(sorted(job.wildcards_dict.items()))): ( + getattr(job.params, "pre", None), job.shellcmd, + job.resources.get("mem_mb")) + for job in dag.jobs + } + assert leaked == [] + + def normalized(jobs, root): + return {key: tuple(v.replace(str(root), "") + if isinstance(v, str) else v for v in value) + for key, value in jobs.items()} + + assert (normalized(dirty_jobs, tmp_path / "dirty") + == normalized(clean_jobs, tmp_path / "clean")) + for _, shell, _ in dirty_jobs.values(): + assert "SP_BLEND_HANDLING" not in (shell or "") + assert "SP_SEG_VIGNET" not in (shell or "") diff --git a/workflow/Snakefile b/workflow/Snakefile index 85b6af482..4749710c8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -196,6 +196,16 @@ if BLEND_HANDLING not in run_config.BLEND_HANDLINGS: f"Invalid blend_handling={BLEND_HANDLING!r}; expected one of " f"{sorted(run_config.BLEND_HANDLINGS)}.") +# The committed inis read SP_SEG_VIGNET and SP_BLEND_HANDLING with noise-fill +# defaults, and only tile.smk's blend_env exports them. Drop any copy the +# launching shell carries, bare or as the APPTAINERENV_/SINGULARITYENV_ forms +# that survive --cleanenv, so a job inherits none: the slurm executor submits +# with --export=ALL from this process's environment, and each job re-parses +# this file before its shell runs. +for _prefix in ("", "APPTAINERENV_", "SINGULARITYENV_"): + for _name in ("SP_SEG_VIGNET", "SP_BLEND_HANDLING"): + os.environ.pop(_prefix + _name, None) + # SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two # invocations of `sp run`, `passthrough` on direct commands (`sp --unlock`, `sp # --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index From 54807b6005211febfd1c9d96356110757989b51d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 01:31:20 +0200 Subject: [PATCH 23/44] Centre SExtractor-mode SEG_VIGNET on X/Y_IMAGE_DBL Under SEG_VIGNET, sextractor_runner runs SExtractor with a copy of the param file that adds X_IMAGE_DBL and Y_IMAGE_DBL; add_seg_vignet centres each stamp at rint(DBL - 1), as SExtractor centres VIGNET, and drops the two columns, so the catalogue is noisefill's plus SEG_VIGNET. The half-pixel tie matching against image minus background goes, and with it the BACKGROUND check image dependency. Co-Authored-By: Claude Opus 5.5 --- .../sextractor_package/sextractor_script.py | 119 +++++++----------- src/shapepipe/modules/sextractor_runner.py | 34 ++--- tests/module/test_sextractor_seg_vignet.py | 101 +++++++++------ workflow/config/cfis/config_tile_Sx.ini | 5 +- 4 files changed, 134 insertions(+), 125 deletions(-) diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index 854872e9c..dbae064d8 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -81,68 +81,51 @@ def seg_vignet_column(seg_vignets): ) -def vignet_centres(x_image, y_image, vignets, image, background): - """0-based centre pixels of SExtractor's VIGNET stamps. +# Double-precision positions add_seg_vignet centres SEG_VIGNET on. +DOUBLE_POSITIONS = ("X_IMAGE_DBL", "Y_IMAGE_DBL") - SExtractor centres VIGNET on the pixel nearest the object's barycentre, - held in double precision. The catalogue's float32 ``X_IMAGE`` / - ``Y_IMAGE`` round to the same pixel except where they land on an exact - half pixel, which hides the side the barycentre was on. For those, each - neighbouring centre is tried and the one whose stamp of ``image`` minus - ``background`` agrees with the object's VIGNET on the most pixels wins. + +def seg_vignet_param_file(dot_param, output_path): + """Write a SExtractor parameter file that also asks for the double + positions. + + SExtractor centres VIGNET on the pixel nearest its double-precision + barycentre, which the float32 ``X_IMAGE`` / ``Y_IMAGE`` hide on exact + half pixels; ``X_IMAGE_DBL`` / ``Y_IMAGE_DBL`` carry it. + :func:`add_seg_vignet` reads them and drops them again. Parameters ---------- - x_image, y_image : numpy.ndarray - 1-based positions as the catalogue stores them - vignets : numpy.ndarray - The catalogue's VIGNET stamps, shape ``(n_obj, size, size)`` - image, background : numpy.ndarray - The measurement image and SExtractor's BACKGROUND check image + dot_param : str + Path to the configured parameter file + output_path : str + Path to write the extended parameter file to Returns ------- - numpy.ndarray - 0-based column of each stamp's centre pixel - numpy.ndarray - 0-based row of each stamp's centre pixel - int - Number of objects on a half pixel + str + ``output_path`` """ - x = np.asarray(x_image, dtype=np.float64) - 1 - y = np.asarray(y_image, dtype=np.float64) - 1 - col, row = np.rint(x).astype(np.int64), np.rint(y).astype(np.int64) - x_tie, y_tie = x % 1 == 0.5, y % 1 == 0.5 - size = vignets.shape[1] - image = image.astype(np.float32, copy=False) - background = background.astype(np.float32, copy=False) - ties = np.flatnonzero(x_tie | y_tie) - for i in ties: - cols = np.floor(x[i]) + np.array([0, 1]) if x_tie[i] else [col[i]] - rows = np.floor(y[i]) + np.array([0, 1]) if y_tie[i] else [row[i]] - candidates = [(int(c), int(r)) for c in cols for r in rows] - valid = vignets[i] > -1e29 - - def agreement(centre): - c, r = [centre[0]], [centre[1]] - stamp = (cut_stamps(image, c, r, size, np.float32(0))[0] - - cut_stamps(background, c, r, size, np.float32(0))[0]) - return np.count_nonzero((stamp == vignets[i]) & valid) - - col[i], row[i] = max(candidates, key=agreement) - return col, row, len(ties) - - -def add_seg_vignet(cat_path, seg_path, image_path, background_path, - w_log=None): + with open(dot_param) as f: + text = f.read() + if text and not text.endswith("\n"): + text += "\n" + with open(output_path, "w") as f: + f.write(text + "".join(f"{name}\n" for name in DOUBLE_POSITIONS)) + return output_path + + +def add_seg_vignet(cat_path, seg_path, w_log=None): """Add the ``SEG_VIGNET`` column to a SExtractor catalogue. The SEGMENTATION check image, whose labels are the catalogue's - ``NUMBER``, is cut on the grid of each object's VIGNET - (:func:`vignet_centres`, :func:`cut_stamps`) and written, int32 and 0 - off the image, as ``SEG_VIGNET`` in ``LDAC_OBJECTS``, which ngmix's - UberSeg blend handling reads. Every other HDU and column is kept. + ``NUMBER``, is cut on the grid of each object's VIGNET, centred on the + pixel nearest ``X_IMAGE_DBL`` / ``Y_IMAGE_DBL`` as SExtractor centres + VIGNET (:func:`cut_stamps`), and written, int32 and 0 off the image, as + ``SEG_VIGNET`` in ``LDAC_OBJECTS``, which ngmix's UberSeg blend handling + reads. The double positions (:func:`seg_vignet_param_file`) are dropped; + every other HDU and column is kept. Parameters ---------- @@ -150,40 +133,33 @@ def add_seg_vignet(cat_path, seg_path, image_path, background_path, Path to the SExtractor FITS-LDAC catalogue, rewritten in place seg_path : str Path to the SEGMENTATION check image - image_path : str - Path to the measurement image - background_path : str - Path to the BACKGROUND check image w_log : logging.Logger, optional Pipeline logger Raises ------ ValueError - If the check images and the image are not on one pixel grid + If the catalogue lacks the double positions """ with fits.open(cat_path) as hdul: hdus = [hdu.copy() for hdu in hdul] objects = next(h for h in hdus if h.name == "LDAC_OBJECTS") data = objects.data - image = fits.getdata(image_path) - background = fits.getdata(background_path) - seg = fits.getdata(seg_path) - if not (seg.shape == image.shape == background.shape): + missing = [name for name in DOUBLE_POSITIONS if name not in data.names] + if missing: raise ValueError( - f"Segmentation {seg.shape}, image {image.shape} and background" - + f" {background.shape} must share one grid." + f"{cat_path} lacks {', '.join(missing)}, which SEG_VIGNET is" + + " centred on; run SExtractor with seg_vignet_param_file." ) - vignets = data["VIGNET"] - col, row, n_tie = vignet_centres( - data["X_IMAGE"], data["Y_IMAGE"], vignets, image, background - ) - del image, background - seg_vignets = cut_stamps(seg, col, row, vignets.shape[1], 0) - del seg + col = np.rint(np.asarray(data["X_IMAGE_DBL"]) - 1).astype(np.int64) + row = np.rint(np.asarray(data["Y_IMAGE_DBL"]) - 1).astype(np.int64) + size = data["VIGNET"].shape[1] + seg_vignets = cut_stamps(fits.getdata(seg_path), col, row, size, 0) + kept = [c for c in objects.columns.columns + if c.name not in DOUBLE_POSITIONS] new = fits.BinTableHDU.from_columns( - objects.columns.columns + [seg_vignet_column(seg_vignets)], + kept + [seg_vignet_column(seg_vignets)], header=objects.header, name="LDAC_OBJECTS", ) @@ -191,11 +167,10 @@ def add_seg_vignet(cat_path, seg_path, image_path, background_path, [new if h is objects else h for h in hdus] ).writeto(cat_path, overwrite=True) if w_log: - centre = vignets.shape[1] // 2 + centre = size // 2 own = np.mean(seg_vignets[:, centre, centre] == data["NUMBER"]) w_log.info( - f"SEG_VIGNET cut from {seg_path} for {len(col)} objects" - + f" ({n_tie} on a half pixel, resolved against VIGNET);" + f"SEG_VIGNET cut from {seg_path} for {len(col)} objects;" + f" centre label = NUMBER for {own:.4f}" ) diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index 1f5beaa11..67c5df345 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -78,6 +78,24 @@ def sextractor_runner( f_wcs_path = input_file_list[-1] input_file_list = list(input_file_list[:-1]) + # SEG_VIGNET (optional, environment-expanded boolean): add the + # SEGMENTATION check image's stamps, on each VIGNET's grid, as the + # SEG_VIGNET column ngmix's UberSeg blend handling reads. SExtractor then + # also writes the double-precision positions VIGNET is centred on, which + # add_seg_vignet reads and drops. + seg_vignet = config.has_option( + module_config_sec, "SEG_VIGNET" + ) and config.getexpandedboolean(module_config_sec, "SEG_VIGNET") + if seg_vignet: + if "SEGMENTATION" not in [key.upper() for key in check_image]: + raise ValueError( + "SEG_VIGNET needs the SEGMENTATION check image in CHECKIMAGE." + ) + dot_param = ss.seg_vignet_param_file( + dot_param, + f"{run_dirs['tmp']}/seg_vignet{file_number_string}.param", + ) + # Create sextractor caller class instance ss_inst = ss.SExtractorCaller( input_file_list, @@ -109,24 +127,10 @@ def sextractor_runner( # Parse SExtractor errors stdout, stderr = ss_inst.parse_errors(stderr, stdout) - # SEG_VIGNET (optional, environment-expanded boolean): add the - # SEGMENTATION check image's stamps, on each VIGNET's grid, as the - # SEG_VIGNET column ngmix's UberSeg blend handling reads. Needs the - # SEGMENTATION and BACKGROUND check images. - if config.has_option( - module_config_sec, "SEG_VIGNET" - ) and config.getexpandedboolean(module_config_sec, "SEG_VIGNET"): - missing = {"SEGMENTATION", "BACKGROUND"} - set(ss_inst.check_paths) - if missing: - raise ValueError( - f"SEG_VIGNET needs the {', '.join(sorted(missing))} check" - + " image(s) in CHECKIMAGE." - ) + if seg_vignet: ss.add_seg_vignet( ss_inst.path_output_file, ss_inst.check_paths["SEGMENTATION"], - input_file_list[0], - ss_inst.check_paths["BACKGROUND"], w_log=w_log, ) diff --git a/tests/module/test_sextractor_seg_vignet.py b/tests/module/test_sextractor_seg_vignet.py index 55ef05ea2..5f5eb13d0 100644 --- a/tests/module/test_sextractor_seg_vignet.py +++ b/tests/module/test_sextractor_seg_vignet.py @@ -1,12 +1,12 @@ """UNIT TESTS FOR SEXTRACTOR MODE'S SEG_VIGNET. ``sextractor_script.add_seg_vignet`` cuts the SEGMENTATION check image on the -grid SExtractor cut each object's VIGNET on, and adds it to the sexcat as the +grid SExtractor cut each object's VIGNET on and adds it to the sexcat as the int32 ``SEG_VIGNET`` column. SExtractor centres VIGNET on the pixel nearest -its double-precision barycentre; the catalogue stores X_IMAGE / Y_IMAGE as -float32, which ties a few positions to an exact half pixel. Those objects -take whichever neighbouring centre reproduces their VIGNET from the image -minus the BACKGROUND check image, bit for bit. +its double-precision barycentre. The catalogue's float32 X_IMAGE / Y_IMAGE +tie a few positions to an exact half pixel, so the runner asks SExtractor +for X_IMAGE_DBL / Y_IMAGE_DBL too (``seg_vignet_param_file``), and +``add_seg_vignet`` centres on them and then drops them. """ import numpy as np @@ -22,22 +22,27 @@ # (NUMBER, x, y): 1-based double-precision barycentres. Object 3's x and # object 4's y sit a float32 ulp either side of a half pixel, so the float32 -# catalogue shows an exact .5 for both but SExtractor rounded them apart. +# catalogue shows an exact .5 for both but SExtractor rounded them apart, and +# each the other way from rounding the float32 value half to even. OBJECTS = [ (1, 20.2, 15.7), (2, 1.3, 48.9), - (3, 30.5 + 5e-7, 30.1), + (3, 31.5 + 5e-7, 30.1), (4, 44.8, 22.5 - 5e-7), (5, 59.9, 1.2), ] +DBL = ["X_IMAGE_DBL", "Y_IMAGE_DBL"] -def _scene(tmp_path): +def _scene(tmp_path, flat=False): + """A sexcat whose VIGNETs are cut as SExtractor cuts them. + + ``flat`` makes the image uniform, so no pixel comparison can tell a tie + object's two candidate centres apart. + """ rng = np.random.default_rng(3) - image = rng.normal(100, 5, (NY, NX)).astype(np.float32) - background = np.linspace(99, 101, NX * NY, dtype=np.float32).reshape( - NY, NX - ) + image = (np.full((NY, NX), 100, np.float32) if flat + else rng.normal(100, 5, (NY, NX)).astype(np.float32)) seg = np.zeros((NY, NX), np.int32) for number, x, y in OBJECTS: col, row = int(np.rint(x - 1)), int(np.rint(y - 1)) @@ -47,16 +52,12 @@ def _scene(tmp_path): y = np.array([o[2] for o in OBJECTS]) col, row = np.rint(x - 1).astype(int), np.rint(y - 1).astype(int) - # VIGNET as SExtractor writes it: image minus background, -1e30 off the - # image and on some neighbours' pixels. - vignets = ss.cut_stamps(image - background, col, row, STAMP, BIG) + vignets = ss.cut_stamps(image, col, row, STAMP, BIG) seg_true = ss.cut_stamps(seg, col, row, STAMP, 0) vignets[(seg_true != 0) & (seg_true != number[:, None, None])] = BIG paths = {name: str(tmp_path / f"{name}-001-001.fits") - for name in ("image", "background", "segmentation", "sexcat")} - fits.PrimaryHDU(image).writeto(paths["image"]) - fits.PrimaryHDU(background).writeto(paths["background"]) + for name in ("segmentation", "sexcat")} fits.PrimaryHDU(seg).writeto(paths["segmentation"]) objects = fits.BinTableHDU.from_columns([ fits.Column(name="NUMBER", format="J", array=number), @@ -65,18 +66,20 @@ def _scene(tmp_path): fits.Column(name="VIGNET", format=f"{STAMP * STAMP}E", array=vignets.reshape(len(number), -1), dim=f"({STAMP},{STAMP})"), + fits.Column(name="THETA_J2000", format="E", array=np.zeros(5)), + fits.Column(name="X_IMAGE_DBL", format="D", array=x), + fits.Column(name="Y_IMAGE_DBL", format="D", array=y), ], name="LDAC_OBJECTS") imhead = fits.BinTableHDU.from_columns( [fits.Column(name="Field Header Card", format="80A", array=np.array(["HISTORY x"]))], name="LDAC_IMHEAD") fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto( paths["sexcat"]) - return paths, seg, seg_true + return paths, seg_true def _add(paths): - ss.add_seg_vignet(paths["sexcat"], paths["segmentation"], paths["image"], - paths["background"]) + ss.add_seg_vignet(paths["sexcat"], paths["segmentation"]) with fits.open(paths["sexcat"]) as hdul: return [h.name for h in hdul], hdul["LDAC_OBJECTS"].data.copy() @@ -89,12 +92,16 @@ def test_float32_catalogue_positions_tie_on_half_pixels(): assert x3 % 1 == 0.5 and y4 % 1 == 0.5 assert np.rint(OBJECTS[2][1] - 1) == np.floor(x3 - 1) + 1 assert np.rint(OBJECTS[3][2] - 1) == np.floor(y4 - 1) + assert np.rint(OBJECTS[2][1] - 1) != np.rint(np.float64(x3) - 1) + assert np.rint(OBJECTS[3][2] - 1) != np.rint(np.float64(y4) - 1) -def test_seg_vignet_is_registered_with_vignet(tmp_path): +@pytest.mark.parametrize("flat", [False, True], ids=["noisy", "flat"]) +def test_seg_vignet_is_registered_with_vignet(tmp_path, flat): """SEG_VIGNET is the check image on VIGNET's grid for every object, the - half-pixel ties included: int32, VIGNET's shape, 0 off the image.""" - paths, _, seg_true = _scene(tmp_path) + half-pixel ties included, even where the image cannot tell the two + candidate centres apart: int32, VIGNET's shape, 0 off the image.""" + paths, seg_true = _scene(tmp_path, flat=flat) names, data = _add(paths) assert names == ["PRIMARY", "LDAC_IMHEAD", "LDAC_OBJECTS"] seg_vignets = data["SEG_VIGNET"] @@ -105,21 +112,50 @@ def test_seg_vignet_is_registered_with_vignet(tmp_path): npt.assert_array_equal(seg_vignets[:, centre, centre], data["NUMBER"]) -def test_seg_vignet_leaves_the_other_columns_alone(tmp_path): - paths, _, _ = _scene(tmp_path) +def test_seg_vignet_replaces_the_double_positions(tmp_path): + """The written catalogue has the columns SExtractor writes without the + double positions, plus SEG_VIGNET, and every other HDU and column as + they were.""" + paths, _ = _scene(tmp_path) with fits.open(paths["sexcat"]) as hdul: before = hdul["LDAC_OBJECTS"].data.copy() imhead = hdul["LDAC_IMHEAD"].data.copy() _, after = _add(paths) - assert after.names == before.names + ["SEG_VIGNET"] - for name in before.names: + kept = [name for name in before.names if name not in DBL] + assert after.names == kept + ["SEG_VIGNET"] + for name in kept: npt.assert_array_equal(after[name], before[name]) with fits.open(paths["sexcat"]) as hdul: npt.assert_array_equal(hdul["LDAC_IMHEAD"].data, imhead) +def test_seg_vignet_needs_the_double_positions(tmp_path): + paths, _ = _scene(tmp_path) + with fits.open(paths["sexcat"]) as hdul: + objects = hdul["LDAC_OBJECTS"] + cols = [c for c in objects.columns if c.name not in DBL] + hdus = [hdul[0].copy(), hdul[1].copy(), + fits.BinTableHDU.from_columns(cols, name="LDAC_OBJECTS")] + fits.HDUList(hdus).writeto(paths["sexcat"], overwrite=True) + with pytest.raises(ValueError, match="X_IMAGE_DBL"): + _add(paths) + + +def test_param_file_adds_the_double_positions(tmp_path): + """The runner's parameter file is the configured one plus the two + double-precision positions, appended after its last parameter.""" + dot_param = tmp_path / "default.param" + dot_param.write_text("NUMBER # running number\nX_IMAGE\nVIGNET(51,51)") + out = ss.seg_vignet_param_file(str(dot_param), str(tmp_path / "out.param")) + lines = [line.split("#")[0].strip() + for line in open(out).read().splitlines()] + assert [line for line in lines if line] == [ + "NUMBER", "X_IMAGE", "VIGNET(51,51)"] + DBL + assert dot_param.read_text().count("DBL") == 0 + + def test_sextractor_caller_names_its_check_images(tmp_path): - """The runner finds the check images add_seg_vignet reads by type.""" + """The runner finds the SEGMENTATION check image by type.""" caller = ss.SExtractorCaller( [str(tmp_path / "image-001-001.fits"), str(tmp_path / "weight-001-001.fits")], @@ -131,10 +167,3 @@ def test_sextractor_caller_names_its_check_images(tmp_path): "BACKGROUND": f"{tmp_path}/background-001-001.fits", "SEGMENTATION": f"{tmp_path}/segmentation-001-001.fits", } - - -def test_seg_vignet_refuses_a_map_on_another_grid(tmp_path): - paths, seg, _ = _scene(tmp_path) - fits.PrimaryHDU(seg[:-1]).writeto(paths["segmentation"], overwrite=True) - with pytest.raises(ValueError, match="grid"): - _add(paths) diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 4dcef22a6..5930b0256 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -113,8 +113,9 @@ CHECKIMAGE = BACKGROUND, SEGMENTATION # Write the SEGMENTATION check image's stamps, cut on each VIGNET's grid, as # the int32 SEG_VIGNET column ngmix's UberSeg blend handling reads (needs the -# BACKGROUND and SEGMENTATION check images). The workflow sets SP_SEG_VIGNET -# under blend_handling: uberseg only (workflow/rules/tile.smk). +# SEGMENTATION check image; SExtractor also computes X/Y_IMAGE_DBL, which +# centre the stamps and are left out of the sexcat). The workflow sets +# SP_SEG_VIGNET under blend_handling: uberseg only (workflow/rules/tile.smk). SEG_VIGNET = ${SP_SEG_VIGNET:-False} # File name suffix for the output sextractor files (optional) From 60ae2345a523971f85bf8e53cf0e363dfe237066 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 01:32:17 +0200 Subject: [PATCH 24/44] Point the stale SEG_VIGNET_PATH message and astra note at SEG_VIGNET Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 3 ++- src/shapepipe/modules/ngmix_package/ngmix.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/astra.yaml b/astra.yaml index b3e5c827d..01fd8bf45 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1457,7 +1457,8 @@ analyses: Covers only pixels shared with a neighbour; defect_fill does not depend on it: defects are filled the same way under every BLEND_HANDLING, and the epoch cuts never count neighbour pixels. - noisefill (the default; the committed config sets no key) gives + noisefill (the default, and the committed ini's unless the + workflow's blend_handling says uberseg) gives weight 0 to the pixels marked -1e30 in the tile VIGNET on other detections' footprints (about 93% of neighbour-footprint pixels on a SExtractor-mode sims tile; every one in catalogue mode, from the diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index e51ca9a3d..6b36ec4e3 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -2349,7 +2349,7 @@ def prepare_ngmix_weights( raise ValueError( "blend_handling='uberseg' requires a segmentation map and the" + " central object_number; none reached prepare_ngmix_weights." - + " Set SEG_VIGNET_PATH on the ngmix run (see" + + " The tile catalogue's SEG_VIGNET column carries the map (see" + " CosmoStat/shapepipe#776)." ) From c704e639c9e39c5cb16b51d20a053c269d19eb3b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 02:51:01 +0200 Subject: [PATCH 25/44] Make UberSeg the default blend handling blend_handling defaults to uberseg in config.yaml and the Snakefile, so a campaign without the key writes SEG_VIGNET in both detection modes and runs ngmix with BLEND_HANDLING = uberseg. The committed inis keep the modules' own noise-fill defaults and the workflow exports the uberseg values, so blend_handling: noisefill plans exactly the previous noise-fill campaign: params_pin_noisefill.json is the previous pin, and a noise-fill run dir resumes with the key set explicitly. The default pin moves in tile_detect and tile_ngmix only. astra's committed blend_handling becomes uberseg, with noisefill the comparison arm. test_committed_blend_handling_fills_defects checks the committed blend_handling x defect_fill pair against prepare_ngmix_weights: the workflow default is the committed blend handling, and under it every defect is zero-weighted and filled. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BbT81hykcvtuefzB3FhYvY --- astra.yaml | 34 ++++++++------- tests/module/test_ngmix_defect_fill.py | 43 +++++++++++++++++++ tests/unit/test_workflow_tile_detection.py | 2 +- tests/workflow/README.md | 5 ++- tests/workflow/params_pin.json | 6 +-- tests/workflow/params_pin_noisefill.json | 43 +++++++++++++++++++ tests/workflow/test_dag.py | 19 +++++--- tests/workflow/test_params_pin.py | 18 ++++++++ universes/committed.yaml | 2 +- workflow/README.md | 9 ++-- workflow/Snakefile | 7 +-- workflow/config.yaml | 16 ++++--- .../config/cfis/config_tile_Ng_template.ini | 5 ++- workflow/config/cfis/config_tile_Sx.ini | 3 +- workflow/config/cfis/config_tile_Uc.ini | 3 +- workflow/rules/tile.smk | 16 ++++--- 16 files changed, 176 insertions(+), 55 deletions(-) create mode 100644 tests/workflow/params_pin_noisefill.json diff --git a/astra.yaml b/astra.yaml index 01fd8bf45..15e1f7d82 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1384,7 +1384,8 @@ analyses: weights through, so a zero-weight pixel's content spreads into the weighted pixels within about a PSF width. DES's ngmixer fills for that reason: "it may be important for codes that take moments or use - FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = + FFTs". Under every BLEND_HANDLING, the committed uberseg included, + and the default DEFECT_FILL = noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise scale otherwise. The tile VIGNET holds -1e30 on other detections' @@ -1457,18 +1458,12 @@ analyses: Covers only pixels shared with a neighbour; defect_fill does not depend on it: defects are filled the same way under every BLEND_HANDLING, and the epoch cuts never count neighbour pixels. - noisefill (the default, and the committed ini's unless the - workflow's blend_handling says uberseg) gives - weight 0 to the pixels marked -1e30 in the tile VIGNET on other - detections' footprints (about 93% of neighbour-footprint pixels on - a SExtractor-mode sims tile; every one in catalogue mode, from the - segmentation map), excluding the off-tile - rows and columns, which are defects, and replaces them with noise; - unmarked - neighbour pixels keep their weight and light. uberseg ignores the - markers and zeroes the weight of pixels nearer a neighbour's coadd - segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) - and leaves the image untouched, as official uberseg does: + The committed choice is uberseg (the workflow's default + blend_handling), which ignores the -1e30 neighbour markers in the + tile VIGNET and zeroes the weight of pixels nearer a neighbour's + coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, + default 1), leaving the neighbour side's image untouched, as + official uberseg does: esheldon/meds get_uberseg returns a weight map (a nearest-segment-pixel Voronoi split). DES Y1's fiducial metacal and the last Y3 config ran on uberseg-weighted stamps with raw neighbour @@ -1485,9 +1480,16 @@ analyses: with simulations (mask_des_y1_uberseg_only, mask_blend_bias_detection). Noise-filling the neighbour side would instead cut the target's own light along an unsheared boundary, a - sharp edge that rings in the FFTs. The recommended comparison arm is - uberseg (weight-only), with defect_fill held equal across arms. - default: noisefill + sharp edge that rings in the FFTs. The alternative, noisefill + (ngmix's own default, and the workflow's blend_handling: + noisefill), gives weight 0 to the pixels marked -1e30 in the tile + VIGNET on other detections' footprints (about 93% of + neighbour-footprint pixels on a SExtractor-mode sims tile; every one + in catalogue mode, from the segmentation map), excluding the + off-tile rows and columns, which are defects, and replaces them with + noise; unmarked neighbour pixels keep their weight and light. It is + the comparison arm, with defect_fill held equal across arms. + default: uberseg options: noisefill: label: Noise-fill the marked neighbour pixels (BLEND_HANDLING = noisefill) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 29d45f0db..9510fa5d3 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -150,6 +150,49 @@ def test_uberseg_defect_in_neighbour_region_is_filled(): assert w_out[0, 3] == 0.0 and gal_out[0, 3] == gal[0, 3] +def test_committed_blend_handling_fills_defects(): + """The committed universe's blend_handling x defect_fill pair is what + prepare_ngmix_weights does: the workflow's default blend_handling is the + committed one, and under it every defect is zero-weighted and filled. + + Failure mode: the committed blend handling skips the defect fill, so the + record claims a fill the default campaign does not run (raw defects, + which astra excludes, reaching metacal). + """ + import yaml + + repo = Path(__file__).resolve().parents[2] + universe = yaml.safe_load((repo / "universes" / "committed.yaml").read_text()) + decisions = universe["analyses"]["shape_measurement"]["decisions"] + blend_handling = decisions["blend_handling"] + defect_fill = decisions["defect_fill"] + assert blend_handling in ngmix_module.BLEND_HANDLINGS + assert defect_fill in ngmix_module.DEFECT_FILLS + workflow = yaml.safe_load((repo / "workflow" / "config.yaml").read_text()) + assert workflow["blend_handling"] == blend_handling + + n = 21 + gal = 1.0e3 + np.arange(n * n, dtype=float).reshape(n, n) + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + flag[3, 15] = 1 + flag[10, 2] = 2**10 + weight[17, 9] = 0.0 + defect = (weight == 0) | (flag != 0) + kwargs = ( + dict(seg=_uberseg_seg(n), object_number=1) + if blend_handling == "uberseg" + else {} + ) + gal_out, w_out, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), bkg_rms=np.ones((n, n)), + blend_handling=blend_handling, defect_fill=defect_fill, **kwargs, + ) + assert np.all(w_out[defect] == 0.0) + assert np.all(gal_out[defect] != gal[defect]), "defects are left raw" + assert np.all(np.abs(gal_out[defect]) < 10.0), "fill is not unit noise" + + # --- prepare_postage_stamps: the fraction cut counts the defect set -------- N_STAMP = 51 diff --git a/tests/unit/test_workflow_tile_detection.py b/tests/unit/test_workflow_tile_detection.py index d42c6c667..4e5d29bf6 100644 --- a/tests/unit/test_workflow_tile_detection.py +++ b/tests/unit/test_workflow_tile_detection.py @@ -200,4 +200,4 @@ def test_blend_handlings_mirror_the_ngmix_module(): if part.strip()) committed = _load("run_config").load( str(REPO_ROOT / "workflow" / "config.yaml")) - assert committed["blend_handling"] == "noisefill" + assert committed["blend_handling"] == "uberseg" diff --git a/tests/workflow/README.md b/tests/workflow/README.md index a6fbfeb04..1c29af03c 100644 --- a/tests/workflow/README.md +++ b/tests/workflow/README.md @@ -44,7 +44,7 @@ The API context remains open while tests inspect jobs and closes before the fixt ## Campaign-boundary pin -`params_pin.json` pins SHA-256 digests of: +`params_pin.json` pins the default campaign (data, psfex, `blend_handling: uberseg`), and `params_pin_noisefill.json` the same campaign with `blend_handling: noisefill`, so a campaign that ran as noisefill stays resumable. Each pins SHA-256 digests of: - `unit_pre()` rendered for every stage; - every rule's shell template; @@ -76,9 +76,10 @@ Apply mutations only to a disposable checkout, run the named test without `--upd | `test_clean_exposure_waits_on_persist_iff_psf` | Drop the persist edge; make it unconditional under fake PSFs; drop vignets consumers; remove the in-scope consumer filter. | | `test_final_cat_merge_reads_every_ready_tile` | Drop one ready tile; append an out-of-scope tile. | | `test_products_use_products_dir_and_run_name` | Rename either merged catalogue or the persist manifest; route products to scratch; derive `CAMPAIGN` from the products directory's basename. | -| `test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg` | Export `BLEND_ENV` under noisefill; drop `blend_env(...)` from either tile_detect or tile_ngmix; export the wrong value; set `NGMIX_SEG_MEM_MB` to 0; give a committed ini's `SEG_VIGNET` / `BLEND_HANDLING` a literal or the wrong default. | +| `test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg` | Make the Snakefile or config.yaml default `noisefill`; export `BLEND_ENV` under noisefill; drop `blend_env(...)` from either tile_detect or tile_ngmix; export the wrong value; set `NGMIX_SEG_MEM_MB` to 0; give a committed ini's `SEG_VIGNET` / `BLEND_HANDLING` a literal or the wrong default. | | `test_unknown_blend_handling_fails_during_parse` | Remove the Snakefile's `blend_handling` check. | | `test_missing_run_fails_during_parse` | Remove `run` from `run_config.REQUIRED`; literal paths must still receive the required-key diagnostic, not a later `KeyError`. | | `test_unit_pre_changes_at_campaign_boundary` | Append a line to `unit_pre`; change one rule's `params.pre`; change one shell; change a rendered thread count. | +| `test_noisefill_plan_changes_at_campaign_boundary` | Export anything under noisefill; change one rule's `params.pre` or shell. | | `test_params_pin_ignores_fixture_root` | Remove fixture-root normalization. | | `test_params_is_a_rerun_trigger_under_both_profiles` | Remove `params` from candide or nibi's `rerun-triggers`. | diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index eeb61d950..eb8e8e157 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -11,7 +11,7 @@ "final_cat_merge": "f6222c98be9777cbab4131d8857bd3817719df9302d0a321c6cd99f67d5bdc76", "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", - "tile_detect": "1b4b0871bf28296f8e5d6855ae47ff2cb84542c93593a5ae5c0ffc3c84c45479", + "tile_detect": "b097127478f57ecaa25312f3bab14701332c3f81412382371e04ccae30806c57", "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", @@ -19,12 +19,12 @@ "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23", "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb", "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111", - "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415", + "tile_ngmix": "5b86694f3bac7b8c3f62d6c1df72b38a8b9173fb69ab53c75d7d70e38f5dc9ff", "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3", "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e", + "sha256": "ae23a4dca4fbb4b0897534fab4a7d84b85a82daddca4ec3ffd706f51ace09f1a", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", diff --git a/tests/workflow/params_pin_noisefill.json b/tests/workflow/params_pin_noisefill.json new file mode 100644 index 000000000..eeb61d950 --- /dev/null +++ b/tests/workflow/params_pin_noisefill.json @@ -0,0 +1,43 @@ +{ + "algorithm": "sha256", + "rules": { + "all": "572122d8d1901e12ff591b30adf405f8920be8183129befececbb819f3392ed8", + "clean_exposure": "22cb76b13a5205d20a02a9bd3b8c8bea5ea2801555e24dd2b84b11145f7e79d9", + "clean_tile": "a5c07b0461526ed407df36a291deb866d4181524b4c8e0fbc3dd047fd9d28479", + "exp_get_images": "71e76ff7f96af1c5d2b85697cc5819e2271911f177a2075253f3b8cfa268c1a9", + "exp_persist": "302e2837542bc1102430c27c81c600b7cda32e8bddcb5fd60d33950987609fff", + "exp_psf": "2c4f6d00f1939ccbf05b4982a0202a0ff92a4373a727f4e00f3aaab4eba03352", + "exp_split": "6e954f8f3d06f44d3f164675912ce27d6216648855d04968f9168bd7d0f2c4fa", + "final_cat_merge": "f6222c98be9777cbab4131d8857bd3817719df9302d0a321c6cd99f67d5bdc76", + "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", + "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", + "tile_detect": "1b4b0871bf28296f8e5d6855ae47ff2cb84542c93593a5ae5c0ffc3c84c45479", + "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", + "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", + "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", + "tile_get_images": "331a67e747f211ebf4c14b946a7af7f9fc9f55243d69d2791d74aecc3ca228c3", + "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23", + "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb", + "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111", + "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415", + "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3", + "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" + }, + "schema": 1, + "sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e", + "unit_pre": { + "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", + "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", + "exp_split": "358fa8bbe59680d4f9839e007b343dd25fed733cf3156dccd30d305d33ac9480", + "tile_detect": "adad5d671fa65dd04433e1c82a725b845635e9d2e368a1e70833ed990df2d55a", + "tile_find_exposures": "c5922fb507f6fd040a179b53fba0818697661016c9688984b6ce249186dc6986", + "tile_get_catalogue": "b124d12252617abb8df8a98d6234ddfd6cc503e19fa72c9a0c48c21559d6f48c", + "tile_get_images": "45b47c44bfeb34ac973b89d4e752c8c82e78028f9f0da379c44627005b45e279", + "tile_make_cat": "7579a52e32e76523c0b77d75c468a47d7f2e2ca0c55865f40e9628a7481cd79d", + "tile_merge_cats": "69cfe94ba941d2ba7b1ce24883961b47b191b80aff381da1865c6bc38de0c354", + "tile_merge_headers": "8590cfa3281c88d43eb8c4fe5760176a13e928a419cbea9bea3a8f88b0634b42", + "tile_ngmix": "b6b75b553a62df54aea6c337dcde1e0cdcd0fe30000e86880c1a4689e42c4a72", + "tile_uncompress": "91a15538491e53ee2b0d52b0472909374270918e4b4c80880c4f5a8329e40161", + "tile_vignets": "a6111cef708aa3c9fb0144b49de9f781eef84d2096ba4f8a3de9bb45b1a3774d" + } +} diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index 67d0282e1..0537e8339 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -163,10 +163,11 @@ def _committed_value(shell, config_dir, option, env, monkeypatch): @pytest.mark.parametrize("detection", ["sextractor", "unions_catalogue"]) def test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg( tmp_path, resolve_dag, monkeypatch, detection): - """noisefill plans exactly the campaign without the knob; uberseg only - adds its two exports to tile_detect's and tile_ngmix's prologues and the - seg stamps' memory to tile_ngmix, and each export turns its committed - ini's option from the noise-fill default to the uberseg value.""" + """A campaign without the knob plans exactly the uberseg campaign. Against + explicit noisefill, uberseg only adds its two exports to tile_detect's and + tile_ngmix's prologues and the seg stamps' memory to tile_ngmix, and each + export turns its committed ini's option from the noise-fill default to + the uberseg value.""" surfaces = {} for blend in (None, "noisefill", "uberseg"): campaign = Campaign(tmp_path / str(blend), "data", "psfex") @@ -183,7 +184,7 @@ def normalized(blend): if isinstance(v, str) else v for v in value) for key, value in jobs.items()} - assert normalized("noisefill") == normalized(None) + assert normalized("uberseg") == normalized(None) rules, noisefill = normalized("noisefill") uberseg_rules, uberseg = normalized("uberseg") assert uberseg_rules == rules @@ -241,10 +242,14 @@ def test_noisefill_ignores_blend_variables_in_the_launch_shell( uberseg variables plans exactly the campaign launched from a clean one, and the parse leaves none of them in the environment jobs inherit (the slurm executor submits with --export=ALL from this process).""" - clean = Campaign(tmp_path / "clean", "data", "psfex") + campaigns = {} + for name in ("clean", "dirty"): + campaigns[name] = Campaign(tmp_path / name, "data", "psfex") + campaigns[name].config["blend_handling"] = "noisefill" + campaigns[name].write_config() + clean, dirty = campaigns["clean"], campaigns["dirty"] _, clean_jobs = _surface(clean, resolve_dag) - dirty = Campaign(tmp_path / "dirty", "data", "psfex") with resolve_dag(dirty, launch_env=INHERITED_BLEND_ENV) as dag: leaked = sorted(set(INHERITED_BLEND_ENV) & set(os.environ)) dirty_jobs = { diff --git a/tests/workflow/test_params_pin.py b/tests/workflow/test_params_pin.py index faeb43158..c97a8177b 100644 --- a/tests/workflow/test_params_pin.py +++ b/tests/workflow/test_params_pin.py @@ -7,6 +7,9 @@ from tests.workflow.params import params_pin PIN = Path(__file__).with_name("params_pin.json") +# blend_handling: noisefill's plan, pinned on its own so a campaign that ran as +# noisefill stays resumable with the knob set explicitly. +NOISEFILL_PIN = Path(__file__).with_name("params_pin_noisefill.json") BOUNDARY_MESSAGE = ( "params.pre is a rerun trigger under both profiles; this change reruns " "every finished unit of a resumed campaign — land it at a campaign " @@ -23,6 +26,21 @@ def test_unit_pre_changes_at_campaign_boundary(psfex_dag, pytestconfig): assert actual == json.loads(PIN.read_text()), BOUNDARY_MESSAGE +def test_noisefill_plan_changes_at_campaign_boundary(tmp_path, resolve_dag, + pytestconfig): + """The same pin for a campaign that sets blend_handling: noisefill.""" + campaign = Campaign(tmp_path / "campaign", "data", "psfex") + campaign.config["blend_handling"] = "noisefill" + campaign.write_config() + with resolve_dag(campaign) as dag: + actual = params_pin(dag) + if pytestconfig.getoption("--update-params-pin"): + NOISEFILL_PIN.write_text( + json.dumps(actual, indent=2, sort_keys=True) + "\n") + assert NOISEFILL_PIN.is_file(), BOUNDARY_MESSAGE + assert actual == json.loads(NOISEFILL_PIN.read_text()), BOUNDARY_MESSAGE + + def test_params_pin_ignores_fixture_root(tmp_path, resolve_dag): """A different temporary campaign directory cannot require a new pin.""" pins = [] diff --git a/universes/committed.yaml b/universes/committed.yaml index f90c5874b..b8f0d5638 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -54,7 +54,7 @@ analyses: psf_likelihood_noise: psf_noise_1em5 megacam_ccd_flip: megapipe_flip defect_fill: noise - blend_handling: noisefill + blend_handling: uberseg epoch_masked_fraction_cut: one_third central_defect_veto: fixed_radii catalogue_assembly: diff --git a/workflow/README.md b/workflow/README.md index 26f00583c..0dde91795 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -34,12 +34,13 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' # tile is detected with SExtractor; the default for image sims). Either way # make_cat writes TILE_UNIQUE_ID = tile_id * 10**6 + NUMBER and ngmix masks # the same neighbours. -# `blend_handling` is ngmix's neighbour treatment: `noisefill` (the default) -# or `uberseg`, under which tile_detect adds the coadd segmentation stamps to -# the sexcat as SEG_VIGNET and ngmix masks neighbours from them. It is chosen +# `blend_handling` is ngmix's neighbour treatment: `uberseg` (the default), +# under which tile_detect adds the coadd segmentation stamps to the sexcat as +# SEG_VIGNET and ngmix masks neighbours from them, or `noisefill`. It is chosen # per campaign: flipping it on an existing run dir reruns tile_detect and the # shape chain for every finished tile, and reclaimed exposures make that -# destructive. An uberseg comparison needs its own `run:`. +# destructive, so a comparison needs its own `run:`; a campaign that ran as +# noisefill resumes unchanged with `blend_handling: noisefill`. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. diff --git a/workflow/Snakefile b/workflow/Snakefile index 4749710c8..113c3d7a0 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -190,14 +190,15 @@ except ValueError as err: # How ngmix treats a stamp's neighbours; tile.smk's blend handling section # carries it to tile_detect and tile_ngmix. -BLEND_HANDLING = config.get("blend_handling", "noisefill") +BLEND_HANDLING = config.get("blend_handling", "uberseg") if BLEND_HANDLING not in run_config.BLEND_HANDLINGS: raise WorkflowError( f"Invalid blend_handling={BLEND_HANDLING!r}; expected one of " f"{sorted(run_config.BLEND_HANDLINGS)}.") -# The committed inis read SP_SEG_VIGNET and SP_BLEND_HANDLING with noise-fill -# defaults, and only tile.smk's blend_env exports them. Drop any copy the +# The committed inis read SP_SEG_VIGNET and SP_BLEND_HANDLING with the modules' +# own noise-fill defaults, and only tile.smk's blend_env exports them (under +# uberseg, the workflow's default). Drop any copy the # launching shell carries, bare or as the APPTAINERENV_/SINGULARITYENV_ forms # that survive --cleanenv, so a job inherits none: the slurm executor submits # with --export=ALL from this process's environment, and each job re-parses diff --git a/workflow/config.yaml b/workflow/config.yaml index 913245b87..171f49ffe 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -20,13 +20,15 @@ input_type: data # campaign's merged catalogues. Required, and set by the run config. # run: smk-g6 -# How ngmix treats a stamp's neighbours (its BLEND_HANDLING): `noisefill` -# noise-fills them; `uberseg` zeroes their weight from the coadd segmentation -# map, whose stamps tile_detect then writes into the sexcat as SEG_VIGNET. -# Chosen per campaign: flipping it on an existing run dir reruns tile_detect -# and everything after it for every finished tile, and reclaimed exposures -# make that destructive -- an uberseg comparison needs its own `run:`. -blend_handling: noisefill +# How ngmix treats a stamp's neighbours (its BLEND_HANDLING): `uberseg` zeroes +# their weight from the coadd segmentation map, whose stamps tile_detect writes +# into the sexcat as SEG_VIGNET; `noisefill` is ngmix's own default, without +# the segmentation stamps. Chosen per campaign: flipping it on an existing run +# dir reruns tile_detect and everything after it for every finished tile, and +# reclaimed exposures make that destructive -- a comparison needs its own +# `run:`. A campaign that ran as noisefill resumes unchanged with +# `blend_handling: noisefill` in its run config. +blend_handling: uberseg # Default settings per input_type, can be overridden by a user-defined run config file. # - tile_detection: allowed are `unions_catalogue` (fetch official UNIONS catalogue diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index dd2119b5f..343708554 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -127,6 +127,7 @@ ID_OBJ_MIN = $NGMIX_ROW_MIN ID_OBJ_MAX = $NGMIX_ROW_MAX # Neighbour treatment: noisefill, or uberseg, which reads the tile catalogue's -# SEG_VIGNET column. The workflow sets SP_BLEND_HANDLING under -# blend_handling: uberseg only (workflow/config.yaml). +# SEG_VIGNET column. The workflow sets SP_BLEND_HANDLING to uberseg under its +# default blend_handling: uberseg, and leaves it unset under noisefill +# (workflow/config.yaml). BLEND_HANDLING = ${SP_BLEND_HANDLING:-noisefill} diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 5930b0256..c9ab0d082 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -115,7 +115,8 @@ CHECKIMAGE = BACKGROUND, SEGMENTATION # the int32 SEG_VIGNET column ngmix's UberSeg blend handling reads (needs the # SEGMENTATION check image; SExtractor also computes X/Y_IMAGE_DBL, which # centre the stamps and are left out of the sexcat). The workflow sets -# SP_SEG_VIGNET under blend_handling: uberseg only (workflow/rules/tile.smk). +# SP_SEG_VIGNET under its default blend_handling: uberseg, and leaves +# it unset under noisefill (workflow/rules/tile.smk). SEG_VIGNET = ${SP_SEG_VIGNET:-False} # File name suffix for the output sextractor files (optional) diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini index 0f1a98cd5..0dd7b23db 100644 --- a/workflow/config/cfis/config_tile_Uc.ini +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -85,7 +85,8 @@ SEGMENTATION = True # Write that map's stamps, relabelled to the catalogue's NUMBER and cut on each # VIGNET's grid, as the int32 SEG_VIGNET column ngmix's UberSeg blend handling # reads. The workflow sets -# SP_SEG_VIGNET under blend_handling: uberseg only (workflow/rules/tile.smk). +# SP_SEG_VIGNET under its default blend_handling: uberseg, and leaves +# it unset under noisefill (workflow/rules/tile.smk). SEG_VIGNET = ${SP_SEG_VIGNET:-False} ## Post-processing diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 96f2b7aa7..a1b0c17b4 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -54,15 +54,16 @@ mode. The converter writes run_sp_tile_Sx/read_ext_sexcat_runner, and the rule l sextractor_runner, the one path every downstream config reads; the manifest is tile_detect.json in both modes, so tile_vignets onwards is the same DAG. -``blend_handling: uberseg`` switches ngmix from noise-filling neighbours to -UberSeg's hard mask, which reads one coadd segmentation stamp per object from +``blend_handling: uberseg``, the default, gives ngmix UberSeg's hard mask +instead of its own noise-fill default; the mask reads one coadd segmentation +stamp per object from the sexcat's SEG_VIGNET column, cut on the grid of the object's VIGNET. tile_detect writes that column in both modes: the converter cuts it from the catalogue's segmentation map relabelled to its NUMBER (-1 on footprints no object claims, 0 on sky), SExtractor mode from the SEGMENTATION check image, labelled by the same run's NUMBER. The switch is two prologue variables and -nothing else (blend_env, below), so a noise-fill campaign runs exactly the -chain it ran without the knob. +nothing else (blend_env, below), so ``blend_handling: noisefill`` runs exactly +the chain the committed inis run with neither variable set. There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe generates no masks: tiles have no instrument flag image of their own, so @@ -484,9 +485,10 @@ rule tile_merge_headers: # --- blend handling ------------------------------------------------------- -# What blend_handling: uberseg exports into a stage's prologue; nothing under -# noisefill, so noise-fill prologues (rerun triggers) are those without the -# knob. The committed inis read each variable with its noise-fill default: +# What blend_handling: uberseg (the default) exports into a stage's prologue; +# nothing under noisefill, whose prologues (rerun triggers) are those the +# committed inis need no variable for. The inis read each variable with the +# module's own noise-fill default: # SEG_VIGNET = ${SP_SEG_VIGNET:-False} in config_tile_Sx.ini and # config_tile_Uc.ini, BLEND_HANDLING = ${SP_BLEND_HANDLING:-noisefill} in # config_tile_Ng_template.ini. Being in the prologue, a flip reruns From cbec30fc70ba6480c7835fd13a6865c9f443c114 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 05:50:00 +0200 Subject: [PATCH 26/44] Measure windowed centroids for UNIONS-catalogue tiles Since #897 the data path takes its tile sample from the UNIONS DR6 catalogue, which has only isophotal barycentres, and read_ext_sexcat copied them into XWIN_*: every tile-side position (stamp centres and OFFSET, the ngmix centroid prior, tile PSF interpolation, epoch membership, the catalogue RA/Dec) was the barycentre, while image simulations get SExtractor's windowed centroid. read_ext_sexcat now measures XWIN_IMAGE/YWIN_IMAGE at every catalogue object: a vectorised numpy port of SExtractor 2.25's compute_winpos (Gaussian window of sigma 2 FLUX_RADIUS / 2.35, 4-sigma aperture with 11x11 edge subsampling, step 2x the weighted mean offset, at most 16 steps, stop below 1e-4 px), on the tile less a SExtractor-like BACK_SIZE 512 / BACK_FILTERSIZE 9 background, with neighbours' segmentation footprints replaced by their mirror image as MASK_TYPE CORRECT does. XWIN_WORLD/YWIN_WORLD come from the tile WCS, 1-based. SExtractor's fallbacks to the barycentre (FLAGS_WIN 1, 2, 4, 8) are reproduced; 16 marks an unmeasurable object and 32 a centroid still moving after 16 steps. FLAGS_WIN joins final_cat.param, and epoch membership reads XWIN_WORLD as in the SExtractor path. On tile 186.307 it reproduces SExtractor's XWIN_IMAGE exactly when SExtractor does not mask neighbours (0 of 41,425 objects off by more than 0.01 px in windows free of zero-weight pixels), and for 97% of objects on the DR6 catalogue path; the barycentre it replaces is off by 0.16 px at the median and 1.4 px at the 99th percentile. astra.yaml's object_position_columns and background_model say so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NsPvmF43rzKrb5eYPNR3Ki --- astra.yaml | 58 ++- .../read_ext_sexcat.py | 118 ++++- .../windowed_position.py | 405 ++++++++++++++++++ .../modules/read_ext_sexcat_runner.py | 7 +- tests/module/data/xwin_sextractor_186307.npz | Bin 0 -> 23951 bytes tests/module/test_read_ext_sexcat.py | 98 ++++- tests/module/test_windowed_position.py | 143 +++++++ workflow/config/cfis/config_tile_Uc.ini | 6 +- workflow/config/cfis/final_cat.param | 4 + .../config/cfis_image_sims/final_cat.param | 4 + 10 files changed, 794 insertions(+), 49 deletions(-) create mode 100644 src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py create mode 100644 tests/module/data/xwin_sextractor_186307.npz create mode 100644 tests/module/test_windowed_position.py diff --git a/astra.yaml b/astra.yaml index 24dd350d3..f664b0863 100644 --- a/astra.yaml +++ b/astra.yaml @@ -440,8 +440,12 @@ analyses: with a finer mesh and a GLOBAL one. ngmix subtracts the exposure BACKGROUND map from each epoch and weights its pixels by BACKGROUND_RMS (shape_measurement.galaxy_pixel_weights). The header - background path is off. Residual sky offsets propagate into - thresholds, fluxes, completeness and shapes. + background path is off. Under tile_detection = unions_catalogue, + read_ext_sexcat re-estimates the tile background with the same mesh + and filter (clipped mode per mesh, median filter, bicubic spline) to + measure windowed centroids (preparation.object_position_columns). + Residual sky offsets propagate into thresholds, fluxes, completeness + and shapes. Values: default_tile.sex#BACK_TYPE = AUTO; default_tile.sex#BACK_SIZE = 512; @@ -780,13 +784,32 @@ analyses: object_position_columns: label: Windowed centroids (XWIN/YWIN) define every position rationale: >- - PSF interpolation sites, tile and multi-epoch stamp centres, and the - catalogue position all use SExtractor's windowed centroid: in pixels - for tile stamps and exposure-side PSF validation, in world - coordinates for tile-side PSF interpolation and multi-epoch stamps. - Windowed, isophotal and model centroids differ systematically for - blends and asymmetric galaxies, and the centroid feeds the position - seed and the centroid prior. + PSF interpolation sites, tile and multi-epoch stamp centres, epoch + membership and the catalogue position all use SExtractor's windowed + centroid: in pixels for tile stamps and exposure-side PSF + validation, in world coordinates for tile-side PSF interpolation, + multi-epoch stamps, epoch membership and the catalogue. It is a + Gaussian-weighted centroid iterated from the isophotal barycentre, + window sigma 2 FLUX_RADIUS / 2.35, and it feeds the stamp OFFSET and + the ngmix centroid prior (shape_measurement.centroid_source). The + UNIONS per-tile catalogue (detection.tile_detection = + unions_catalogue) carries only the barycentre, so read_ext_sexcat + measures the windowed centroid itself at every catalogue object: a + numpy port of SExtractor 2.25's compute_winpos, started from X_IMAGE, + Y_IMAGE with the catalogue's FLUX_RADIUS, on the tile less a + SExtractor-like background (detection.background_model), with + neighbours' segmentation footprints replaced by their mirror image + as MASK_TYPE CORRECT does. On DR6 tile 186.307 it matches the tile + SExtractor's own XWIN_IMAGE to 0.01 px for 97% of objects, the rest + being blends whose SExtractor masking the segmentation map does not + fully record; the barycentre it replaces is off by 0.16 px at the + median and 1.4 px at the 99th percentile. Where SExtractor would + fall back to the barycentre (FLAGS_WIN 1, 2, 4, 8), or nothing can + be measured (16), the position is the barycentre; FLAGS_WIN, in the + final catalogue, records it, and 32 marks a centroid still moving + after 16 steps, kept as SExtractor keeps it. The tile VIGNET is + centred on the barycentre, as SExtractor's is. Image simulations + (sextractor) take the same quantity from SExtractor. Values: VIGNETMAKER_RUNNER_RUN_1.POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE; VIGNETMAKER_RUNNER_RUN_1.COORD = PIX; @@ -795,11 +818,24 @@ analyses: VIGNETMAKER_RUNNER_RUN_2.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD; VIGNETMAKER_RUNNER_RUN_2.COORD = SPHE; config_exp_psfex.ini#PSFEX_INTERP_RUNNER.POSITION_PARAMS = - XWIN_IMAGE,YWIN_IMAGE. + XWIN_IMAGE,YWIN_IMAGE; + config_tile_Uc.ini#READ_EXT_SEXCAT_RUNNER.WORLD_POSITION = + XWIN_WORLD,YWIN_WORLD; + config_tile_Sx.ini#SEXTRACTOR_RUNNER.WORLD_POSITION = + XWIN_WORLD,YWIN_WORLD. default: xwin_windowed options: xwin_windowed: - label: Windowed centroids everywhere + label: >- + Windowed centroids everywhere, measured by ShapePipe on + UNIONS-catalogue tiles + catalogue_barycentre: + label: The UNIONS catalogue's isophotal barycentre under the XWIN names + excluded: true + excluded_reason: >- + Image simulations measure windowed centroids, so data and + simulations would differ in the centroid the stamp OFFSET and + the ngmix centroid prior start from. stamp_positioning_and_padding: label: Nearest-pixel stamp extraction with zero padding rationale: >- diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py index ff4f58cc8..fa698d312 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py @@ -1,7 +1,8 @@ """READ EXTERNAL SEXCAT. Convert an ASCII SExtractor-format catalogue to FITS-LDAC for use in -ShapePipe tile processing. +ShapePipe tile processing, measuring the windowed positions the catalogue +lacks on the tile image. :Author: Martin Kilbinger @@ -10,6 +11,9 @@ import numpy as np from astropy.io import ascii as asc from astropy.io import fits +from astropy.wcs import WCS + +from shapepipe.modules.read_ext_sexcat_package import windowed_position as wp def _build_ldac_imhead(img_header): @@ -200,7 +204,74 @@ def _extract_vignets(image_data, x_pos, y_pos, stamp_size, seg=None, return vignets -def _build_ldac_objects(cat_data, vignets): +def measure_windowed_positions(cat_data, image_data, img_header, seg=None): + """``XWIN_*``, ``YWIN_*`` and ``FLAGS_WIN`` for a catalogue without them. + + The catalogue is the sample; its isophotal barycentre (``X_IMAGE``, + ``Y_IMAGE``) and half-light radius (``FLUX_RADIUS``) start and size + SExtractor's windowed centroid, measured on the tile image less a + SExtractor-like background map (``default_tile.sex``: ``BACK_SIZE`` + 512, ``BACK_FILTERSIZE`` 9). Pixels exactly 0 carry no data in a + MegaPipe tile and are left out of the background. Given the relabelled + segmentation map, neighbours are masked as SExtractor's ``MASK_TYPE + CORRECT`` masks them. Where the window fails, the position stays the + barycentre and ``FLAGS_WIN`` says why + (:mod:`windowed_position`). World positions come from the tile WCS, + 1-based as SExtractor's are. + + Parameters + ---------- + cat_data : astropy.table.Table + Catalogue with ``NUMBER``, ``X_IMAGE``, ``Y_IMAGE``, ``FLUX_RADIUS``, + and optionally ``A_WORLD``, ``B_WORLD``, ``THETA_J2000`` + image_data : numpy.ndarray + Tile image + img_header : astropy.io.fits.Header + Tile header, for the WCS + seg : numpy.ndarray, optional + Segmentation map relabelled to ``NUMBER`` (:func:`relabel_seg`) + + Returns + ------- + list of (str, numpy.ndarray, str) + ``XWIN_IMAGE``, ``YWIN_IMAGE``, ``XWIN_WORLD``, ``YWIN_WORLD``, + ``FLAGS_WIN`` columns with their FITS formats + + @sc [decision:preparation.object_position_columns] + """ + missing = {"X_IMAGE", "Y_IMAGE", "FLUX_RADIUS"} - set(cat_data.colnames) + if missing: + raise ValueError( + f"The catalogue lacks {sorted(missing)}, which the windowed" + + " centroid needs." + ) + x = np.asarray(cat_data["X_IMAGE"], dtype=np.float64) + y = np.asarray(cat_data["Y_IMAGE"], dtype=np.float64) + wcs = WCS(img_header) + cxx = cyy = cxy = None + if {"A_WORLD", "B_WORLD", "THETA_J2000"} <= set(cat_data.colnames): + cxx, cyy, cxy = wp.isophotal_ellipse( + cat_data["A_WORLD"], cat_data["B_WORLD"], cat_data["THETA_J2000"], + wcs, x, y, + ) + measured = image_data - wp.sextractor_background( + image_data, good=image_data != 0 + ) + xwin, ywin, flags = wp.windowed_positions( + measured, x, y, cat_data["FLUX_RADIUS"], seg=seg, + number=np.asarray(cat_data["NUMBER"]), cxx=cxx, cyy=cyy, cxy=cxy, + ) + ra, dec = wcs.all_pix2world(xwin, ywin, 1) + return [ + ("XWIN_IMAGE", xwin, "D"), + ("YWIN_IMAGE", ywin, "D"), + ("XWIN_WORLD", ra, "D"), + ("YWIN_WORLD", dec, "D"), + ("FLAGS_WIN", flags, "I"), + ] + + +def _build_ldac_objects(cat_data, vignets, windowed=()): """Build LDAC_OBJECTS extension from an astropy table. Parameters @@ -209,6 +280,8 @@ def _build_ldac_objects(cat_data, vignets): Catalogue data read from the ASCII SExtractor file vignets : numpy.ndarray Array of shape ``(n_obj, stamp_size, stamp_size)`` + windowed : sequence of (str, numpy.ndarray, str), optional + Extra ``(name, array, FITS format)`` columns, the windowed positions Returns ------- @@ -231,23 +304,8 @@ def _build_ldac_objects(cat_data, vignets): fmt = f"{arr.dtype.itemsize}A" fits_cols.append(fits.Column(name=colname, format=fmt, array=arr)) - _aliases = { - "XWIN_IMAGE": "X_IMAGE", - "YWIN_IMAGE": "Y_IMAGE", - "XWIN_WORLD": "ALPHA_J2000", - "YWIN_WORLD": "DELTA_J2000", - } - col_map = {c.name: c for c in fits_cols} - for alias, source in _aliases.items(): - if source in col_map and alias not in col_map: - src_col = col_map[source] - fits_cols.append( - fits.Column( - name=alias, - format=src_col.format, - array=np.array(cat_data[source]), - ) - ) + for name, arr, fmt in windowed: + fits_cols.append(fits.Column(name=name, format=fmt, array=arr)) vignet_col = fits.Column( name="VIGNET", @@ -281,7 +339,10 @@ def make_ldac_from_ascii( The input columns, ``NUMBER`` included, are copied unchanged; a ``VIGNET`` column (postage stamps extracted from the tile image) is - added to ``LDAC_OBJECTS``. Given the catalogue's segmentation map, which + added to ``LDAC_OBJECTS``, and, when the catalogue has no windowed + positions, ``XWIN_IMAGE``, ``YWIN_IMAGE``, ``XWIN_WORLD``, + ``YWIN_WORLD`` and ``FLAGS_WIN`` measured on the tile + (:func:`measure_windowed_positions`). Given the catalogue's segmentation map, which shares the tile's pixel grid, the map is relabelled to the catalogue's ``NUMBER`` (:func:`relabel_seg`) and used to set neighbours' pixels in each ``VIGNET`` to ``BIG``, as SExtractor does. @@ -345,8 +406,23 @@ def make_ldac_from_ascii( number=np.asarray(cat_data["NUMBER"]), ) + windowed = [] + if "XWIN_IMAGE" not in cat_data.colnames: + windowed = measure_windowed_positions( + cat_data, image_data, img_header, seg=seg + ) + if w_log and windowed: + flags = windowed[-1][1] + w_log.info( + "Measured windowed positions; FLAGS_WIN counts: " + + ", ".join( + f"{v}={c}" for v, c in zip(*np.unique(flags, + return_counts=True)) + ) + ) + ldac_imhead = _build_ldac_imhead(img_header) - ldac_objects = _build_ldac_objects(cat_data, vignets) + ldac_objects = _build_ldac_objects(cat_data, vignets, windowed) hdul_out = fits.HDUList([fits.PrimaryHDU(), ldac_imhead, ldac_objects]) hdul_out.writeto(output_cat_path, overwrite=True) diff --git a/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py b/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py new file mode 100644 index 000000000..4c7659063 --- /dev/null +++ b/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py @@ -0,0 +1,405 @@ +"""WINDOWED POSITION. + +SExtractor's windowed centroid (``XWIN_IMAGE``, ``YWIN_IMAGE``, +``FLAGS_WIN``), measured on a tile image at given starting positions, and the +background map SExtractor subtracts before measuring it. + +The centroid is a numpy port of ``compute_winpos`` in SExtractor 2.25.0 +(``src/winpos.c``), the version the tile SExtractor run uses; the +background is an approximation of its ``makeback`` (``src/back.c``). Both +work on 0-based pixel indices internally and take and return 1-based (FITS) +positions, as SExtractor does. + +:Author: Cail Daley + +""" + +import numpy as np +from scipy.interpolate import CubicSpline + +# SExtractor 2.25.0, src/winpos.h and src/growth.h. +WINPOS_NITERMAX = 16 # maximum number of steps +WINPOS_NSIG = 4 # measurement radius, in window sigmas +WINPOS_OVERSAMP = 11 # subpixels per pixel side on the aperture boundary +WINPOS_STEPMIN = 1e-4 # stop once the mean offset is below this, in pixels +WINPOS_FAC = 2.0 # step = WINPOS_FAC * mean offset (exact for a Gaussian) +GROWTH_MINHLRAD = 0.5 # minimum half-light radius, in pixels + +# FLAGS_WIN bits. 1, 2, 4 and 8 are SExtractor's own, each of which makes it +# revert to the isophotal barycentre; 16 and 32 are ShapePipe's. +FLAGS_WIN_SINGULAR = 1 # windowed second moments are singular +FLAGS_WIN_NEGATIVE_MOMENT = 2 # a windowed second moment is negative +FLAGS_WIN_NEGATIVE_FLUX = 4 # windowed flux is not positive +FLAGS_WIN_WANDERED = 8 # centroid left the isophotal 1-sigma ellipse +FLAGS_WIN_UNMEASURED = 16 # no measurement: barycentre not finite or off image +FLAGS_WIN_NOT_CONVERGED = 32 # still moving after 16 steps; position kept +FLAGS_WIN_FALLBACK = ( + FLAGS_WIN_SINGULAR + | FLAGS_WIN_NEGATIVE_MOMENT + | FLAGS_WIN_NEGATIVE_FLUX + | FLAGS_WIN_WANDERED + | FLAGS_WIN_UNMEASURED +) + +# Pixels per object and chunk, bounding the working arrays (~100 MB). +_CHUNK_PIXELS = 2_000_000 + + +def _natural_spline(nodes, at, axis): + """Natural cubic spline through equally spaced nodes 0, 1, ..., n - 1. + + Outside the nodes the end cubic continues, as in SExtractor's + ``subbackline``; a single node is constant. + """ + if nodes.shape[axis] == 1: + return np.repeat(nodes, len(at), axis=axis) + grid = np.arange(nodes.shape[axis]) + return CubicSpline(grid, nodes, axis=axis, bc_type="natural")(at) + + +def _mesh_mode(pix): + """SExtractor's background estimate for the pixels of one mesh. + + ``backstat`` clips at the mean +- 2 sigma, and ``backguess`` iterates a + 3 sigma clip about the median, then takes the mode estimate + 2.5 median - 1.5 mean unless the distribution is too skewed, when it + keeps the median. SExtractor does both on a quantised histogram; this + does them on the pixel values. + """ + mean, sigma = pix.mean(), pix.std() + pix = pix[np.abs(pix - mean) <= 2 * sigma] + lo, hi = -np.inf, np.inf + for _ in range(100): + sel = pix[(pix >= lo) & (pix <= hi)] + med, mean, sigma = np.median(sel), sel.mean(), sel.std() + new_lo, new_hi = med - 3 * sigma, med + 3 * sigma + if new_lo == lo and new_hi == hi: + break + lo, hi = new_lo, new_hi + if sigma > 0 and abs(mean - med) / sigma < 0.3: + return 2.5 * med - 1.5 * mean + return med + + +def sextractor_background(image, back_size=512, filter_size=9, good=None): + """Background map as SExtractor's ``BACK_TYPE AUTO`` estimates it. + + The image is cut into ``back_size`` meshes (the last row and column + smaller); each mesh with at least half its pixels good gets the clipped + mode of :func:`_mesh_mode`, a bad mesh the mean of its nearest good + meshes; a ``filter_size`` median filter, narrowed symmetrically at the + edges, smooths the mesh map (``BACK_FILTTHRESH`` 0); and a bicubic + natural spline through the mesh centres interpolates it to every pixel. + + Parameters + ---------- + image : numpy.ndarray + 2-D image, shape ``(ny, nx)`` + back_size : int, optional + ``BACK_SIZE``, the mesh side in pixels + filter_size : int, optional + ``BACK_FILTERSIZE``, the median-filter side in meshes + good : numpy.ndarray, optional + Boolean map of the pixels to use (SExtractor: weight above + threshold); default all finite pixels + + Returns + ------- + numpy.ndarray + Background map, float32, shape ``(ny, nx)`` + + @sc [decision:detection.background_model] + """ + ny, nx = image.shape + nbx, nby = (nx - 1) // back_size + 1, (ny - 1) // back_size + 1 + if good is None: + good = np.isfinite(image) + back = np.full((nby, nbx), np.nan) + for j in range(nby): + for i in range(nbx): + cut = np.s_[j * back_size:(j + 1) * back_size, + i * back_size:(i + 1) * back_size] + pix = image[cut][good[cut]] + if pix.size >= 0.5 * image[cut].size: + back[j, i] = _mesh_mode(pix.astype(np.float64)) + + ok = np.argwhere(~np.isnan(back)) + for j, i in np.argwhere(np.isnan(back)): + if len(ok) == 0: + back[j, i] = 0.0 + continue + d2 = ((ok - (j, i)) ** 2).sum(1) + back[j, i] = back[tuple(ok[d2 == d2.min()].T)].mean() + + half = filter_size // 2 + filtered = np.empty_like(back) + for j in range(nby): + hy = min(half, j, nby - 1 - j) + for i in range(nbx): + hx = min(half, i, nbx - 1 - i) + filtered[j, i] = np.median( + back[j - hy:j + hy + 1, i - hx:i + hx + 1] + ) + + fy = np.arange(ny) / back_size - 0.5 + fx = np.arange(nx) / back_size - 0.5 + rows = _natural_spline(filtered, fy, axis=0) + out = np.empty((ny, nx), np.float32) + step = max(1, 4_000_000 // nx) + for y0 in range(0, ny, step): + out[y0:y0 + step] = _natural_spline(rows[y0:y0 + step], fx, axis=1) + return out + + +def _aperture_fraction(dx, dy, raper2): + """Fraction of each pixel's 11 x 11 subpixel grid inside the aperture. + + ``winpos.c`` counts the subpixel centres ``(dx + i/11, dy + j/11)``, + ``i, j`` in ``-5..5``, with squared radius below ``raper2``; this counts + them row by row in closed form. + """ + half = WINPOS_OVERSAMP // 2 + sub = np.arange(-half, half + 1) / WINPOS_OVERSAMP + rem = raper2[:, None] - (dy[:, None] + sub) ** 2 + s = np.sqrt(np.clip(rem, 0, None)) + lo = np.maximum( + np.floor(WINPOS_OVERSAMP * (-s - dx[:, None])) + 1, -half + ) + hi = np.minimum(np.ceil(WINPOS_OVERSAMP * (s - dx[:, None])) - 1, half) + count = np.where(rem > 0, np.clip(hi - lo + 1, 0, None), 0) + return count.sum(1) / WINPOS_OVERSAMP**2 + + +def _blanked(label, own): + """Whether pixels of ``label`` are blanked while measuring ``own``. + + SExtractor (``MASK_TYPE`` BLANK or CORRECT) sets every detected pixel + to -BIG as the scan passes it, and pastes an object's own pixels back + only while it measures that object: every footprint but the object's + own is blanked, including those no catalogue object claims. + """ + return (label != 0) & (label != own) + + +def _iterate(image, mx, my, sig, seg=None, own=None, + n_iter_max=WINPOS_NITERMAX): + """Run ``compute_winpos``'s loop for objects sharing one stamp size. + + ``mx``, ``my`` are 0-based starting positions and are not modified. + With ``seg``, pixels :func:`_blanked` for the object take the value + of their mirror image through the current centre, or 0 where the + mirror is off the image or blanked too: SExtractor's ``MASK_TYPE + CORRECT``. Returns the final + 0-based positions, the windowed flux ``tv``, the windowed second + moments of the last step, and whether each object stopped on the step + criterion. + """ + ny, nx = image.shape + n = len(mx) + mx, my = mx.astype(np.float64), my.astype(np.float64) + invtwosig2 = 1.0 / (2.0 * sig * sig) + raper = WINPOS_NSIG * sig + raper2 = raper * raper + rintlim2 = np.clip(raper - 0.75, 0, None) ** 2 + rextlim2 = (raper + 0.75) ** 2 + half = int(np.ceil(raper.max())) + 2 + offs = np.arange(-half, half + 1) + + tv = np.zeros(n) + mom = np.zeros((3, n)) + converged = np.zeros(n, bool) + active = np.arange(n) + for _ in range(n_iter_max): + if active.size == 0: + break + amx, amy = mx[active], my[active] + xs = np.rint(amx).astype(np.int64)[:, None] + offs + ys = np.rint(amy).astype(np.int64)[:, None] + offs + # The C box: [(int)(m - raper + 0.499999), (int)(m + raper + # + 1.499999)) clipped to the image; (int) truncates, and the + # clip makes truncation and floor agree. + r = raper[active][:, None] + in_x = (xs >= np.clip(np.trunc(amx[:, None] - r + 0.499999), 0, None)) + in_x &= xs < np.clip(np.trunc(amx[:, None] + r + 1.499999), None, nx) + in_y = (ys >= np.clip(np.trunc(amy[:, None] - r + 0.499999), 0, None)) + in_y &= ys < np.clip(np.trunc(amy[:, None] + r + 1.499999), None, ny) + + dx = (xs - amx[:, None])[:, None, :] + dy = (ys - amy[:, None])[:, :, None] + r2 = dx * dx + dy * dy + area = (r2 < rextlim2[active, None, None]).astype(np.float64) + ring = (r2 > rintlim2[active, None, None]) & (area > 0) + if ring.any(): + k, iy, ix = np.nonzero(ring) + area[k, iy, ix] = _aperture_fraction( + dx[k, 0, ix], dy[k, iy, 0], raper2[active][k] + ) + area *= in_y[:, :, None] & in_x[:, None, :] + cy = np.clip(ys, 0, ny - 1)[:, :, None] + cx = np.clip(xs, 0, nx - 1)[:, None, :] + pix = image[cy, cx] + if seg is not None: + lab = seg[cy, cx] + blank = _blanked(lab, own[active, None, None]) & (area > 0) + if blank.any(): + k, iy, ix = np.nonzero(blank) + # (int)(2 m + 0.49999 - x), truncating as C does. + x2 = np.trunc(2 * amx[k] + 0.49999 - xs[k, ix]) + y2 = np.trunc(2 * amy[k] + 0.49999 - ys[k, iy]) + x2, y2 = x2.astype(np.int64), y2.astype(np.int64) + inside = (x2 >= 0) & (x2 < nx) & (y2 >= 0) & (y2 < ny) + x2c, y2c = np.clip(x2, 0, nx - 1), np.clip(y2, 0, ny - 1) + mlab = seg[y2c, x2c] + usable = inside & ~_blanked(mlab, own[active][k]) + pix = pix.astype(np.float64) + pix[k, iy, ix] = np.where(usable, image[y2c, x2c], 0.0) + locpix = area * np.exp(-r2 * invtwosig2[active, None, None]) * pix + + atv = locpix.sum((1, 2)) + pos = atv > 0 + safe = np.where(pos, atv, 1.0) + dxpos = (locpix * dx).sum((1, 2)) / safe + dypos = (locpix * dy).sum((1, 2)) / safe + tv[active] = atv + mom[0, active] = (locpix * dx * dx).sum((1, 2)) / safe - dxpos**2 + mom[1, active] = (locpix * dy * dy).sum((1, 2)) / safe - dypos**2 + mom[2, active] = (locpix * dx * dy).sum((1, 2)) / safe - dxpos * dypos + mx[active] += np.where(pos, dxpos * WINPOS_FAC, 0.0) + my[active] += np.where(pos, dypos * WINPOS_FAC, 0.0) + done = pos & (dxpos**2 + dypos**2 < WINPOS_STEPMIN**2) + converged[active[done]] = True + active = active[pos & ~done] + return mx, my, tv, mom, converged + + +def windowed_positions(image, x_image, y_image, flux_radius, seg=None, + number=None, cxx=None, cyy=None, cxy=None): + """SExtractor's windowed centroid, started from given positions. + + For each object, a Gaussian window of sigma + ``2 * hl / 2.35`` (``hl`` = ``|flux_radius|``, at least 0.5 pixel) + weights the image within ``WINPOS_NSIG`` sigmas, the aperture edge + subsampled 11 x 11; the centre moves by ``WINPOS_FAC`` times the + weighted mean offset, up to ``WINPOS_NITERMAX`` times, until that + offset is below ``WINPOS_STEPMIN``. Given the segmentation map, + neighbours' footprints take the mirror image of the object's side, as + SExtractor's ``MASK_TYPE CORRECT`` does. Where SExtractor would revert + to the isophotal position (``FLAGS_WIN`` 1, 2, 4, 8) this returns the + input position, and so it does where there is nothing to measure (16). + A centroid still moving after the last step keeps its position, as + SExtractor's does, and is flagged 32. + + Parameters + ---------- + image : numpy.ndarray + Background-subtracted image, shape ``(ny, nx)`` + x_image, y_image : array_like + 1-based starting positions: SExtractor's ``X_IMAGE``, ``Y_IMAGE`` + flux_radius : array_like + Half-light radius, SExtractor's ``FLUX_RADIUS`` for + ``PHOT_FLUXFRAC`` 0.5, in pixels + seg : numpy.ndarray, optional + Segmentation map on the image grid, 0 for sky, labelled with + ``number`` (:func:`read_ext_sexcat.relabel_seg`) + number : array_like, optional + Each object's label in ``seg``, required with it + cxx, cyy, cxy : array_like, optional + Isophotal ellipse (``CXX_IMAGE`` ...) for the flag-8 test; without + them bit 8 is never set + + Returns + ------- + numpy.ndarray + ``XWIN_IMAGE``, 1-based + numpy.ndarray + ``YWIN_IMAGE``, 1-based + numpy.ndarray + ``FLAGS_WIN``, int16 + + @sc [decision:preparation.object_position_columns] + """ + ny, nx = image.shape + x0 = np.asarray(x_image, dtype=np.float64) + y0 = np.asarray(y_image, dtype=np.float64) + fr = np.asarray(flux_radius, dtype=np.float64) + n = len(x0) + flags = np.zeros(n, np.int16) + xwin, ywin = x0.copy(), y0.copy() + + ok = np.isfinite(x0) & np.isfinite(y0) & np.isfinite(fr) + ok &= (x0 >= 0.5) & (x0 < nx + 0.5) & (y0 >= 0.5) & (y0 < ny + 0.5) + flags[~ok] |= FLAGS_WIN_UNMEASURED + sig = np.maximum(np.abs(np.where(ok, fr, 1.0)), GROWTH_MINHLRAD) * 2 / 2.35 + half = np.ceil(WINPOS_NSIG * sig).astype(np.int64) + 2 + + own = None if seg is None else np.asarray(number) + tv = np.zeros(n) + mom = np.zeros((3, n)) + for h in np.unique(half[ok]): + members = np.flatnonzero(ok & (half == h)) + size = (2 * h + 1) ** 2 + for c0 in range(0, len(members), max(1, _CHUNK_PIXELS // size)): + idx = members[c0:c0 + max(1, _CHUNK_PIXELS // size)] + mx, my, tv[idx], mom[:, idx], conv = _iterate( + image, x0[idx] - 1.0, y0[idx] - 1.0, sig[idx], seg, + None if own is None else own[idx], + ) + xwin[idx], ywin[idx] = mx + 1.0, my + 1.0 + stopped_on_flux = tv[idx] <= 0 + flags[idx[~conv & ~stopped_on_flux]] |= FLAGS_WIN_NOT_CONVERGED + + mx2, my2, mxy = mom + measured = ok + flags[measured & (mx2 * my2 - mxy * mxy < 0)] |= FLAGS_WIN_SINGULAR + flags[measured & ((mx2 < 0) | (my2 < 0))] |= FLAGS_WIN_NEGATIVE_MOMENT + flags[measured & (tv <= 0)] |= FLAGS_WIN_NEGATIVE_FLUX + if cxx is not None: + dx, dy = xwin - x0, ywin - y0 + outside = ( + np.asarray(cxx) * dx * dx + + np.asarray(cyy) * dy * dy + + np.asarray(cxy) * dx * dy + ) > 1.0 + flags[measured & (dx * dx > 1) & (dy * dy > 1) & outside] |= ( + FLAGS_WIN_WANDERED + ) + revert = (flags & FLAGS_WIN_FALLBACK) != 0 + xwin[revert], ywin[revert] = x0[revert], y0[revert] + return xwin, ywin, flags + + +def isophotal_ellipse(a_world, b_world, theta_j2000, wcs, x_image, y_image): + """``CXX_IMAGE``, ``CYY_IMAGE``, ``CXY_IMAGE`` from the world ellipse. + + Maps the isophotal ellipse (``A_WORLD``, ``B_WORLD`` in degrees, + ``THETA_J2000`` east of north) to pixels through the local Jacobian of + ``wcs`` at each 1-based position, and returns its inverse-covariance + coefficients in SExtractor's convention + (``CXX x^2 + CYY y^2 + CXY x y = 1`` on the 1-sigma ellipse). + """ + x = np.asarray(x_image, dtype=np.float64) + y = np.asarray(y_image, dtype=np.float64) + eps = 0.5 + ra, dec = wcs.all_pix2world( + np.concatenate([x, x + eps, x]), np.concatenate([y, y, y + eps]), 1 + ) + n = len(x) + cosd = np.cos(np.deg2rad(dec[:n])) + # d(east, north)/d(x, y), degrees per pixel; east = +RA. + dra = (ra[n:] - np.tile(ra[:n], 2) + 180.0) % 360.0 - 180.0 + ddec = dec[n:] - np.tile(dec[:n], 2) + jac = np.empty((n, 2, 2)) + jac[:, 0, 0], jac[:, 0, 1] = dra[:n] * cosd / eps, dra[n:] * cosd / eps + jac[:, 1, 0], jac[:, 1, 1] = ddec[:n] / eps, ddec[n:] / eps + + theta = np.deg2rad(np.asarray(theta_j2000, dtype=np.float64)) + u = np.stack([np.sin(theta), np.cos(theta)], -1) + v = np.stack([np.cos(theta), -np.sin(theta)], -1) + a2 = np.asarray(a_world, dtype=np.float64) ** 2 + b2 = np.asarray(b_world, dtype=np.float64) ** 2 + cov_w = (a2[:, None, None] * u[:, :, None] * u[:, None, :] + + b2[:, None, None] * v[:, :, None] * v[:, None, :]) + jinv = np.linalg.inv(jac) + cov = jinv @ cov_w @ np.swapaxes(jinv, 1, 2) + icov = np.linalg.inv(cov) + return icov[:, 0, 0], icov[:, 1, 1], 2 * icov[:, 0, 1] diff --git a/src/shapepipe/modules/read_ext_sexcat_runner.py b/src/shapepipe/modules/read_ext_sexcat_runner.py index 17e23163d..60249fd73 100644 --- a/src/shapepipe/modules/read_ext_sexcat_runner.py +++ b/src/shapepipe/modules/read_ext_sexcat_runner.py @@ -32,8 +32,11 @@ def read_ext_sexcat_runner( a FITS-LDAC catalogue compatible with downstream ShapePipe modules. The inputs are the catalogue and the tile image, then the catalogue's segmentation map if SEGMENTATION = True, then the WCS log if - MAKE_POST_PROCESS = True. With the segmentation map, neighbours' pixels - in VIGNET are set to -1e30; the map itself is not written out; the + MAKE_POST_PROCESS = True. The windowed positions (XWIN_*, FLAGS_WIN) + are measured on the tile image at each catalogue object. With the + segmentation map, neighbours' pixels in VIGNET are set to -1e30, and + the windowed centroid masks neighbours as SExtractor does; the map + itself is not written out; the only output is the FITS-LDAC ``sexcat.fits``. MAKE_POST_PROCESS runs the multi-epoch post-processing that adds per-exposure HDUs. """ diff --git a/tests/module/data/xwin_sextractor_186307.npz b/tests/module/data/xwin_sextractor_186307.npz new file mode 100644 index 0000000000000000000000000000000000000000..7f5242d60ba41e405353d8fb2dc6fdbffae84362 GIT binary patch literal 23951 zcmc$`bC4#Bmbcqw+qP|V**3at+qP}nRb962>auOy#@#dL^vpSXcFf%G&Np{5q9XsP zhF>_4W4y{u}Z)eS0$#Q%4;$D?Jk$0Er&!-i<$BnMC4p^jcIUf%EByxkD3Tx^=x z*hCKlrD`q->~cMPtJ1`l*8AmDWed-C@NB7jENZHo-q#<0MczIvegNnAZ zaYFSK&87Q^(Bi-bJz3CaO6NMIdPsdoRJbU^Hgk(}IIjVM0lusy)9`R7KOPUNPZ`SJ z>~jzhm?GjWB?l;LolH$XCTiA5qEuK1;qrb-Ml69r|2035>+jV};H^yglBVMOZIr=u zbxSn)B~!5N212fkO|oz$tD?s(J?lqLz-ZkbR2$^JoH?_$aMgOm%g%qL2` zb5w$AB$6cySJ1sItm4-(4p_!Bf_d3Ff@_8?YC6f{^(-ZjnNAAOs|@Iw=KxG_IPJ4rln>9jPPBMtbVtr z@|N!8*VDqLr(KZTUKTsdiG4^(A@D!bd|^dxLh#pHDeg-^OV?+5Z>=ODmfSHe7JRI# zHm%`KaS$@OTg70?MG73jKn$I&!xNi1C7H;q#FR&Urtj!Q6X(()%Bv-N30+8d+`0ju zzpiU)L|{$mBERw5u~|l_!TKTTZ1C6sG|PGrysFTqc`RgvnrP{D9(clQ$>T%PCHv|P zsLOFWW1(!Wy;4~ z7hNCewktp^QA*$&-MC2@$srNKjwKWZc8fbUv6gg*hV0=f`)+wu>WMty%-$m=B!bn+ zHU9v%A-(FKj53&B&4NpNICb@NfbK$F2?N86GC+xdTYl&Own>&8M(qZ_?nDy6J}e0| znVsletp(mMLFOZu`b7;@p#X zs;2qNClc(KjeW(A<6^5JNyft|u}heqn`mdo#DaFa#-x~CO%Xxk#Y14}mw~OK2M)H+ zxi7HC0DP?*CefA8niqS&tem>E3qqIJ~M~2rds>!M8VJ;8l?RzBRu}rBCRo6&H3T z?56?_Xw~Ss6~)#g4E0PZNg9B`w-w^3owFZ$=Vn~&Y|*CQxFk*8b|i{@UeOgBsRJP} zS&g)t*=FxZD^5;sohwvrL32#a{H77r-ru&H@#Vtx-Eh4--h6+)9H5a?wgnM<;ufTg)EX z-suJycGT%fp+=k0uvaikksvbOi0~NCxa*kc?owLE)|8t>7gZd`tn}n`wMXD?V@F*I zd=hN(Bs9&k3|^^41z$CSLv+FrqqNuO*3!o}Z%LBisK;Vq1437f-fnOK=aO3520+0` z|Bff<@}4)9nAtigWrJaenF)r@uZ3D)q!==z98*|y_R^rJ3jQhRgJ8TssY}d;fDCn% z>j2O>I}9UR$}1UMs8HD+do?C_FdBY-+@U|ul^qgVMqO3E;$O{m(Q9+|VHS0|V;SC37$!7Ccwlt(?-$A=mRy-uY->Tg%+dXY*R;wPbG{-r zjM-6k-cG@HIX%D9ztXtc+D2+fInn zZZ#pBy`J*f{DcS$9CM)Bv&gk;gMrD=^sJI|mk3dG}`q#kH3-k(3`|q8e#z`F76-vg%X` zEe)Q;Zf41)P{j${dBab9b~c=<+8T$+i;?(wj5f`!4`MX&*0URiD%NSAcOn2Vn~^T5 z9YwRk-02?aZXqYw(S|W^?U3dnz26GUtz1Rkhie3Kx4rEXFXkeT+8o=BJQn?fAmJMU zUkA^s`XdB~&xtVXR(Ij~A_@VD5;R`73*Q}*Pq2FS_XfB!yFL^o@|Ya3-2x2ao-Hsp zkqy%F`8{@sEtm~+dtz%HnY{7Vq0B57P$q^U>!4QnaPmQ_=E5PPU|-aDDEv zipPMnd6NAWd?-vfa{o(D1`hcI*F)3&e!d#lS%b@V51d>|5c=Y9!V_!rV+`xPgW+$2@m1`+4@v=dQ7QE#lG#7Ozu zbBgGcFK*8=F6oCBLTl}7lv{>dQu}4wl+G$-CC56OPv>ctp zct}JS$#Du9kZ?2;V$?#w7fg|2PDqjBbWX@dNJBu{U;uzfU9q3;N@8uWp6)f}!$n}t z=9Zk-%K}4%lGg=Vv14I zOtIg-{c}ltHIXpCOZta-)xlBE-tl))sedm<^*=AlIPbqIYKZe+3qt-z3yN?0TygL% zA1B+?{}@d5l8NKbV4bc|Zok2h{=L^muC_)7jz)&R+5RZ#Uk~N~E}O`|v*nnasry)P z(4R-NFBg{`{F$sQtoHIZ8R9>XSz4OeI{e#^ANhA6x&Hzz|NnKYb}cliked7 zkM(}lTL3v9puZfUEk9^rq@1ANpDy86x`|xhHb1(&zwTha`@=qeoWY&1*G^q+S((-j zC=Kg}#g)mVPJV@*7cr%KBIt-ZKo#`9yzmk;^FbyPYXezqX$du1gNi7MQ>w@bHEA&o zYQ_h2cPj3XLHY3Oq+qLTT!vGA&w5$WS~IQLC6)6bm#$RhM&vF@v*lp*YY4*b4?Y2k z#laQ4uP@#fTm!5qtuWBLd?_q-%i7+n=%*)Z!WAn)}vBVo7uj+Y)@l9!dh)_`k4kS(iZ$5)N*zgfc2Dy-1LFA%5E zzBQoJUtPoDqG-D&%_k0pWcl^+ji4U+q>u>ly-k9RYaGt^xYX%>$iuCz^$RlF>LG9y6zcQ#CzmmH!=X$xHBB3E^?jX1c1qoHOEkYM$f#actD_!`+7N z7C-pFkQNUTdY+RqTg&s*jy1XG#pdr>)>bmFYK{s(B$s zX1xjn%=%S0U4W2R;Q~740Q~qeLA_y$J3`Qc$lvuF)pTjb%t6kM4(aM5%^(H3yvrU{ zCDR=6;KkT059E4}VH9YaGwZm$2C}ZL*2Ox8yzJDYu!)Wq3JT9Dg>Z&XqN?>wwX54r zVDhoL8lk~K44e2gK5tenxU^=HZ=X$j$BdJX{vy(+M1gc=D@W<1Ot(xh(%D(tlpmLo z?YG@9A-odsO0O#kz^0`EY@BqcXko^5K4Q~1m?0{?O)9|2lD$c}nAAkq17$0hxOT6P zby5v4ut>_-mdc}{deg&%VW>sTgryA{e0VfnbwiXM{y9B063M*Icr+VxF1IC$1-S?7 zJ6hzzPOKLxt4glGuT9!ZX!d&X?&)soirOhp+IzK#YT(r8E$5!(Z?hQ~H-G?Y=ta|& zj9BvG%;+gJ9G^Bh7h;~pki23*F4yYn$6bDE%zeqz;>Lx@_{u>#aIo_rPlg;k*~6F9 zxDO`_Lg-o)7U0~IX_GzJP;azj2p598IqK}aV_#ZL7Lk$Ij6#3YGZn9}R> zde@qpSlFQ^qdfwY#VQx;!^iKokT2O7l^<^TRPI;q4>64w2%e4UeAELJc%F2^c%LT( z_1A&YwK#);(jvIcl-7Ky1EXSxYsyb2@p#N%XM9a-Y%2N2LehO`yJ2o%ZdDRN+xIeF zZHlf~h$c*kTqK#NlaYSkNg@C~lIR&f*~`!p@+FzQbUo$5tY^wY)~gI7o73TnmS%5! zo4Jy%8H*;OY#6!>Ij1(g)$yBczKc1H&FTla#~(bQGB0Eoe4Jg4$YT3})E_3~Y*_}B zN-j(%8cR;Jh4Am4kb@H~Dq7?*L}u?c5g;h!Z?3=9SPsEw(kg*zdIL1f?YB6N17T;r zf_l$;e480Gcg$`A%)zMx8NT2GE>?`*d*wlhNZ%4lbYBeB|GfJKRH@KYH2ZDc3#L>s z2@AzbJg+&#Ag`wtIa6tgzq*JhSo~+e@LYobcnyWpqo`4v!zA?Qu5!NVbT{(Vdsbgt z4yV3a7NyRkSXNnAwg^z-t-Ogegr;{zNjROan4KryfR{7z;j4bw}yu1iriK z5GKvUuUyhvY|@jH{E6_Od1VQNEfDnSK9^O`B2CUWY)=&BuIPYi!Gtzyr9&Mr^7g02 zc(@p21}hhuE(Gq@w>nnVpY2IJ-unI|YwHZl4WPVi&AvOYfb5&8aE$RRu_xt%)gB|$ zYPYARhB<<)S*5;wDXh=-M-1Lj(F0A3SL1{GO!C^Xfr<|%(>RY$;=ekPGw=4%E6laW zJ{r?|f6!n~NowmJdf1EgspuFN-$-A(uUWZrv?4>Q2TxbBC7yTZgQ$qw@Xb4$tv>Cd zH?Wy0?$w#zmNENfscnufE{WDn`#J_A2BRBUN>zf9&5=#sv=HU{62E2XI~WqFNm+94 zvoPn)^Z;_U8wRp>%#&uiERTp+_z&TJXV>IR(tkqNW>t@AZ?YzgbT{BR!fxK6C7+CEj@4*$%|j z6}_AH8DKbSkH%?r361~Lsk=gGl_WUN8TxTChu;i<~lBB$#5t{W6ZR9 zHh8fGZ>0_^lT561aMxunp#jx@!G?S-3q-V0%DGnJaT$-z;eQy5DL(_C+*|0CJczFm zHD?GXg5-aS;e2V0~zbMKG|=^N`K>iKR+f|$AKE}%2wvo z>5tlOoY0gu_S(I}jCp(|i2+x9T7bV@RWh&Kkq*BW3$?wD_)oeQq?b$*4bt=E@Zprd+0+}m!S z?CPW7rA40T9nj`M6PPNRYNh_O(i&#!iJTWa*`Zc~-TZ^u_`R|u6cscdEEv|x<-^Nfb&;RkN<#K*g?(K|_Db$fpm)~oL$9a{3E6CA+uZa) zp)j8Yf`X8vQ-aA2^77Fdo-c&4bvTBr$)YXo@pSY2z?B{Kkohj&z$-7>w(@7FubvS@ zRF&1a)HDHVLA*m820Hk6vU<{f=B$A}E&v)=_VOe1f~E5_{z%r9wnc{|Tx-oK;u5QU z@@H(WjT;dDR_AI1ZX4+!2i>~i@(^Ms)&r?FQv_zqsabDzL5J^e{1Tiw^2^&n_E{ys zPgm+a*JXNK6)&;u&S|zZqm0vO7s5gtc5L^V^SaXI^ckl)5T5_{&!FDXl{>H;krZ&gEm~eo4UxX<4%ys-I zZ)XDydz5C$;6}F#r;djqAT4ct7+b>@xASFTxmws_C&8p0t7k$M5r+bB+5KtG8U#iK zpzW2~K*1z)sYSvpwf75BHnz->3b*v`3Zyobdn59_6Oz?7q!F`>o=)>Dkj|0R2fq%9`_ye`EcNqWw)m z{^#AXDtOO24EMDS7TQFcnx1^g`Op0-K!W)>Nfxs3f51xZc>`%^Mh0NMh6YCunVPYWEtNZ*&KiQR~1j8bhy90WetduP*`(x(Gmp@N}5wMqa{ zge9V!_Agd;5p`&E0Z(Lon}r)rctN#+zo0Il!u#M@=pODN`9j?~-3Pmza|(p5^{mNY zD)(;0rgC}nzw2kdjzlL6c#PO~X~I_GCQ4Iiq`B)h3S`9^7i(-wyi=kDZ44q4lQu_d z@c9J7?i4Ar@Zm(GJ0~+<1!AMhmTS5ZSmwTs&jHVnTsT0wSIHeJ*~Q4~>u+=<)`#G2 z?e+5z;no~2pE5jXN)jnr$KR2`7=>WQd?ni`tEq;cH9DwS;|fs0_^x=guGdBC%JAu^ zoUF>RAX--a5o8`wh)EPb)OzhpT+FCHDKKwx9Ma4wB{$c5K~;T(a}cwAa+Rs~a8mOA zRM|2QnC0?e>_${9+Z5-VUKX{@ZJJl^K}*gajgJPDG%Gd}ZB>4~ zT?So@2>50D96)Xn0_OYxTfX;f`2bAH4$e?X5RK8ZVyqW-oe~mWiRl6ZWx1iJI}nvJ zUHG|gU!%<1G#9G&-?c$z#$c9Cj2=L-XHy8okx*t8luh(RA7<<}=xV47edq0)?|xz1 zBowKBd+FvC<t z1LfwN^X1qQMa;F>e=B)bW-1hp;v&91vo7;#CrPT~w$H%cQE#Glx1ie_u66vv4nGUy z>s6;F`NVy+lQjb7XlKO(!6JPv;m9y}vcSyaU5_ed?tIrJk^sY0Hc>Nt&a|Jp=?59o z`Ti_NIGABWvd9oJj#lJwbk<&a)I^Q6wleCVL6g1 zoi;k;GQrwcD<1~cSf+1FBc|66bJWKh&u<461+%RpQXMva+E5knDu>-J>-NkohLOor zUlSY>vm~`-Iy`kG8U}}L?PRqB@G?4*aS_LC$7VnK(EOo5jAjA8t{l%V=WWSTwg#Ms zx&`Fci_w+P2lW=Sv38yh2@t4(W)GY*o7}G^?8;S*6!KB9K)J1F#p`VHf(RGl} zw?d6|*;;V`I!t-l&5B|a<^&Qc-r!_%hZ^%3val|;emhYwY>=*)!YYV$Qe0->84yI% zEY*)Fy3KWs8!H}$Fwap25BRx6PtgKb4BZ|4gY3{CWP<#2u?y+#A&WwaDqJWY`0(}h zr)lkQ0Rt0T=q82!2F6RWvuM`U6M(^#U zy)J^Q_Y$QYM8U4|J=YJHmt0Sm#A?JdqIL`q#80D8D~|dazZL!PFes7l5`@w0c6liS z@^t%vQ6*vhqF4IMa~~+D?4i5s8F!^x%>dZYiM1x20Dajx0B3_!7XETtgHD%jUlPM@{@-Kse5tkOb2;i=pD50SCwCN1pQ1`cw8wpbY8qZ9j94{xe zUtuYl5zqW8)u+8fgfaflha^8*m%xK@If&ReNAcV=VAs~L38)XFufx3nN|(s)j@Kj4ha$og-r<$v5BVDlDf3|De$0V-FMlx+}1(}vUN zW`J?#nm)1&6vNrK^O$dMNQQ^tRm+j`J+_^;`;U zp4_6jbLs}}a?AWMN^-#eH-a73lGVI7*S>|Ix4p2Z~X$Fbt*FF^_E8^SxR%k?>mJ5?4&EyRT) zlockgdtymE;NBnadW~rCclvSt6Sq{J$y8|$w}>YK;)jWCfu3v|gWcj4JBqAu2U|O~ z11@H6M?CfZ6mAndMpXd6u!j9>dnzKQH5+DE=I-HPU|d=p$JuH*zkRvHNm~St_XTlu z0PRNKosae&C+>sd)U=!ZbaGG?KSG{>FuYNdie24m8u5oFpZo_zgHV0Wys<3*+`}Mo zNkL#`0Y5xjN6s5nDIz-shEj@(kZ1_j{T(6H&!3aIbyW!9iI+v;C1}U1$t<41hRjcg zak!fch=UcI%-&TWY6>qjzA=zVJ@yge-hG}}>{()YmVUJRuuAgJrO3kP>r>4+alT$O z-7L{*?~uG*KD3@{TT)5QiYL#5N%$7nvTg3!9m}N*2R!YQ6A3PQ3|UZWNoOE!RMbVU zPo#$R`}BcJ0H5a1C`FO#b7B{am>BP9S=TIEnVT(J3D61er-YC$kZb&(V`}13(()8C4{DML>!WV%sig zbVcl>eH)IB`pqIjYJwBS==QGASRxJtz`SvLWpLlxT^Oe(==9Q@m^wWS`$9giF=W_F zcUBOb#%YgtaVu;Hm(qT)duT(Ov~}&=C)$^7Uj=#2G{8!$9nY5RB8!DL>y=?5t%pSb zdV5P^w`y01^h2&F8In!ecm6CDwF_M(QGPa_qn*`y#Za$)jB9ij-P?`{*38?4udE)~ z1Fg)tnMGQ2ohvCY+$7hnd+m%xA^W8TmtfGJcI#^331&f`WMqW3MzB7PkPav;cBaON ztKbDTyYO_<9~6;TOO`dDb+@hbK=`Qh&j#6`!s_>Xs7C**q53am((XSOkNza>i+ADWojIkGdTkh&0?Wt}29?FB&(rOP(J7&$5JeVkGw){V#psFFo_W zy$Sq>EhYSN{@%BIOc}F)V_+fzRdE zGco-SdsHdGF{0m$*ni3RKP{iMGg}T9jn02OT==uP4ew7a{73o3{VTlx1rLQVNg4g! zGOWK~V*FTM$^dO{rz-x?qusmw0S{J5h2!%htV?tAd=P%>EV%^ef4 z^z2~S;Mj6O-ugDvm7gt}Ht4;qdEe*}BRrX-L^`r_9Rym;jLREt4^4(%341cA3Qk@+ zqwuDFIp|Wtp59TymhQX$tt zx3<3GUzf%cD9O|vR4FG=HlntX;~~`KdWt{!L20o6$~roau3GarTJp)vmTKl1>WqiD zH^(76a=HtD-*C_~v^(0DPo>Y(?gq+>an>(2MO?Fwa2bnGDMMG)RL7~qAk5vMJ0zLR zgfj#3qo8=Ab5AJOQz3+Mt;lsQkab7tiTzv^JYak?rRLF@XU$DH#l4kq>sc92csM0* z3KpSx?u3~CQ-{A(4j3t%^M`N2cO7)jv^91VUr8M@vq_qK;b!FPCwrGoP6q%w*tvky zQ=#_V6>Hb85NH+IXIic`70?u|Y$Z>ct4Uw*QLV9mTk>oWgkw^xJ%140=SBrNXWs$K}-V^{~Y zuAOSVqa=uqk(uOHpQ(MeBKo1sa@CpDVd?xk@;JhxxPAWb-5){6!ovzptoL?l}D`LzOTK$u6KlF&1YXZ1r ziPC=NA%Pj+BjBY`nJFi0wopxbQt~0;q&d2rzGW!m@~B~Ei%DH=?8U@+!y;K>0m%ZA zmn-~j%&|}L=q6V88Zf~%0=D#iod138r(Dg z<~Jh|V}Zmc{XDu`mpa83fRx9P-Qx^(E`!hS?v1&8BiuCz*hjCr6Sl^ zx&UKCM%y>}9&&d^%xH*8&qc|6FIfPP#Zv52CwupuIQO^wnp!$w+Oh z(6!_~_Jag#obgNVNm82P&~-#2<_-{Lqm(oW4%#6cbV}K{2F^%1&S0S}T=b1iP(Ip` z1etyxMDiF2B3I4w_q!XIW~@QJd$OP9OrP)|9O^sm<<)CihEcu)CEm!RtWhb$+x_r2 zUV2=RE4pXl)awfWGTAQ~>Hx*Me5(ZT6tD^KmWQ5xVZ07i%->ny?Atb>alCvQI$?X= z7UyFcYu@S`jsZ5@RlK$oij(`$18O`nW;AHxH!X`l0{V#gQQ3{T*h=O{jcv^RbTvDg zr7GUC1j@GX6mUYLn7zLUGgnxy=kb9~9bN@I;l(1?k$Lkb+cxTXrarm@P2+n13XMwZ z4$T%*DLPm&>e|+W9$_ef&wmR793f@8>J1sr5)#-9iOEHp2frSz6!(0AIp3li))|5n zG}XpyRzz(;F#1|UzzC@MgiX+@m$95~@j>tS#(+F>2K9^SpjCTF$y(kAJ1RYSkNIxt z+O8RZwrY_di^HV?K0QrjCHvv@#%ZyK<}Ll1p( z#zE)=3&ekJ*-?U5cdV{^Js3|%VkbJYS~nwWJX7+)c4Qyo&fcyl|1M&ux5y{A!G|qJ zO@oHHdl3L@Z5aB|WX=wWh}O%b93S$SLn9||9413pKt>JCQK*cFJaWJ*0V!{%$MhVX zV4Kex2lU|`zF1;dUacE_9J(og&V??~5e-Vl?dHdsDFE1&d4jCiH@dF{WEVHM(9NF{ zHYAjH&l|~zWrj@G12U*Byl^U~Ir$XbIEDLjUG$&DIM5Y##?RU)NCubs6?zQDX&HHX zi=Oi>*`STQObnCTl&LX1)6*};sJNTN#>tmn^kaJ5$zPfQj~Y0E4LS=C+00Q}ZAjo5 z#`N#cmEq7dhc~Fr3_sjSV$8VRAU2U6=e(3ryag+hP&b#u5b9<5y7b8A7{&c*6k1yr z76Ir^s|I8%L2527(QH@BOp6Lc=+@f_lCpv?Kega(xU*DPqw_AOkaRDM(89rC2ojvH zh9tu+7`)%5eM?>Db!IoGv1zFy#a{uak7za5Y9R?73~%hxfn|%8#Y{aI%w7l5H`q@NiT$nJg#-O%9!c=fe(l~wxLK2QAkHgF1lrbrX=U`jsc683Nf;vR=V7|`l;KF)8Oez| z5^Hb@EB8?@JmNYhP{9CTpXs4Dv+LC2Gwg&uf(NyhK1*MOy_7N{4*+ zUGf*b+p@+g%NBplyI|z0`wjcO6;?+THGl?;?EJ4czVaXROEjMlj|;W7#)%Z>+~vN{ zuo|YeU;L;K0seC=KI(3~K`|-~{NJ|>QuD?}Z41acDBmSRy*vVyCC+w2n7$AI-V%xXA_=f9PEg@^ z*>&64DNHA(uRoolFH!-^j)m)1FYU?D3c2fc<|<~37Dk?Hi%`x|ExB*A5JKP$bvS9nEd(P=K|W=nNQX+&AB%MC(=1xHrX@nuLa&b4ICwt23Y#Bq)~I-{M3 z;t?lu#pb0GpQ`U&{;m`FvyGHIdw~AH4HwzCNH?+wuWD8vWXaPjb-KovZ2f*%&=`(D zZ{D7jW*Ycw*#k6}6~TV^Nc>bmR?haEcJZ_kz}Z>aFr(W_{}7i22Jr>Z9gQI)IOivT z!v*hgR|QDU4^3%2rnNkx=;a+}y5uYdjMIsvo2X-gQc~H0FMxlJ3A{M$m47=A#$P(m zKjZ|6V1F|m*Z+1-ko?EeIr$%#&Y%RkfdcjqaaAdrS&=}w31OCAQo5am0Ho~1s(5AY zWd#XJ-{yqdCgu6K{~Q-Q?R&@kF5$1W?Jx6!|Eqgx_U-(mZ}}Mej{Nt4d1dFYf3jVb z9HWHah<|0fj5_~+BnHxVmsbdjF1js*&KczbO8|d@9A9vG`wjV5sm33W|I?8{75ti@ zbJ6P4#Ubvq71W>j{xLGZ`^%5&|3YTaNW^&kyJ@I@Ksgw>>KN-;TI%Z=Sp0Kb5Qy-% zyN2VByz5_%EY%aXrHduuLkgio1;cRS`~fH0I75kiVv1%0MT}W$`FJ4#VZt;j!bHfe zsaA;qNQ9*0MWN8a?Zsj0^prsp%yKG7|; zVT2h!(xE6wvBU*ssM=^)4Nc|{e;5QuCyABdZXmS-wIwiebjr9rI$6`8rQ(Dj8Xax# zja#zc8m4M2mx=IJrW7Pe2k07n;s}_$M;Kd0S{GD)Y?d+mbL%_9BwT~%#X zwY{VD&_G@4=7=4?r^mw#)$VXfID$4JEhAE7vwB zpz;%BgDa4^na-IwwP)&(XIH574MZRi`0$!mU{gPe_l^NDKqp#yt7^armNJ1@>A^MU zhsNZ#N2G=1m_Ic%ZWxhT8lj z)N9Y!KBhVZ2lXBVr;n=odqY!+ING~>Rg}`5#{e|51s#%-Dh>$!U#!+{CU6OROFfrW zq^^0czU5p@nzb%5$VmfO~cOzUe4!=o`TdDBEjsI0ue zAodESmJ7W1uO=R>7h-Iel@g>WT}Qt2`*+8h$$ntt?8lh)dM|=({7q=`s|$Y8Q80sd zE`%#(`WDV@W4=X!Tf64eXZ$4QjHl|zaqJ8Tr(RZ^8IPAhsxS0l++>ml`lFbap_4x7 z4nI*v+5d2~K@4L523uVPG@f{?fzv)3L+2-IB^++Rz7VUM_C1_uZ+f_2sIKugK@nw{ zDTMt^^}60PNJ^Pgx$e~vRP*+RpDx8rA(r6+(qY?vrSisE6Kkz01f*HSKoDN3h5Bm+=s{WC}^2=_CB@ODN0j zBJtdcqk;f=2Mu1b^dux9VHuHd@_5~WE@zEBmBToadsha#FFe*AJwPhfjCsc0iaiGn zEQ8HF{`k^-BOj}|; zxa(M28S_V_eT9+dYM)razlt*iawEFTYjQIHHEJNiC zB@PpHkf1SE9?x>X^vRylR}tazSit<4JeCQ}PgSuV1QM-jCkN93`l#g+_mB3P?)l}g zscX66wuwY0PmOI#79$|Ee1Xr=B<{siDMeqb^YyI|z4h7Tx>_nQaQFS3DQy~R>2GlHI z^C@SA5_gVqj~qyct;qosf-nCbI=aWn1;``Ap_!joBBfQpHCI!_GUNORaMj#=<2i7< zzVi6)pj?)oeM10H&y+ty|9CZk!DK9;$D=K( zcXeo>iAxas1~ElxgL8@F$=#4S1KGGKPGD@cyKY8X&fZd}U_M`7Jxd0zi#x0?8$ZDwj+aJJ)0V8az3zxg@jU7>upuJS<1_hSh_qF zoxo%qUktAKLf-Wd&?iY!xI$)FmWLG#5>-HafZppNs*glEK*701=B+22Ha_~B^-fe-cag5gJZ)XWOF53Li2#BG&hsQDhw&#A00%kwvL>p2_vh< z6y!E|N&HsOuX}jI1HY&!auZcHj)81zdtr3DRTwmWmdp+nhQ;?aB)VMtfLrot$M9j* zgzH>cwr#aII{1q*zto)*N?pjS$9P3v2|~5UcaLd#u=_=kXbYe_F&)2;HZ8h?zR)DZ zreCuWSKHwMeAOo{=DzcfuQ#;|Q)~MGdJT}#+bwZ6-5yw5GhbreWSfakLER}%jnKco zc(E*b4}cbQhsCZQU0f@3`57HtGi8KQlS9cGc-7Cvew8u^=#$92l1c+@WdgNZz1HwJ zAVr(2?`At-o$Q>Hi_azjCw`<3VJOLpJROqU@9tAxV`KD&y&z_~jUpc^aYkCPPL8XS zh8a>A>rRn~qN^PCs#AxYG#XE5*99gHl~4NzcPeev=DaB(`%j^Q^cls$94mCj-*$d9 z*M4rWXcly&R-V-w>$83JM1Z~IYTtv}Trz@O>`IAoo3+Ko6?CU8(`B}9WOr#1fmVDv-g&Eve}a>R zk+WHl#7wCwBQgEeGUOU=XUuR?r#M0&6JvbGWKWs5L60jF!R!jC0Io)_ZG@!0`B$C3 za-`mC_O-Yb+KFv1t}`$b&BSH?<`VdQts$s{X~Q5WI9uKq#@x1gK{QlrdmdrsykdCQ8>3 zZ-`T&pA)f<`zU<#+-B;`PY1E>}fSb$lpzSy@QYtO)Op zt(oK(xXQ!ktyk?pSI$o51eI;A)M6PAs#UxLKp` z4K~AVuQx5GAdwxHA1B$SIVz+=*3q+BUVVc^^Ys=wJS(2Aj&}Pkmz!L2HHdcpR~aoLy^n+45T44vo1oS#=kR$85h}(WFVJyBTj8pp(5LA8?oC_o5nL zD%$o_64k*8H*ps5Yr>^a*ZU&8JaOH<`3+Qc01~lLOt)gSwd|L^7^Yhz%yM$0QaZt_ z;~6r7oe?|JeV~9_0JxOdf?lFc4AJ4dpohsb7mq@rf=%eroOMil@}szym@y`;K&4u< zS}<=W286kxjg;X9s3hE883-o;d&zr)Gz}~3e&XV!TdK#yV9GY>{IB3nCq0>B1D|lAy#+K;gl5afVyjdxLFi?vlKCXg7@s{&*`sAy6NK%(fD4kmv#ewihW(++;w`-Z^FirZ0F16W{+pW z4x#80z+pRe2e-7OGv-$1>E~?F>GKBSSK_iyz+4X=(q@#WGMWzBsU21ALY}^dMfb3&MzrW0^f&)K}sMFfNo=U(-Qwu37DQwE1NhTYO z6TAd7Wp~=ct}2&5g6R(vWznZE2W0jkJFwQ;q;`^I)4K z1XRH%Xc%RdDE96Z82sWr>W?{Px$j#z7!0u8wmFocov7Z|X#qL$GLUkb343=k6Z^pr zl*pu)0{$QttP?V&srhvCDlUOR!sJ=tUwg5r+M%2qtd^!O^i)644XVa z*6Plcw*v!qSu`lfVx4QN6yVO$354o$Gj$hD=*nqFo%MkI>|_e+IR!7InmDc((&Fko z;-O80FZlxc=KxRhP^tKL0P!#10RN`|;vcd*z`wtA%Isgmhd*9I_5bw^1w%uekV8ZN znB9p0K={V#e?%%V1Ss@~fuzXw(0CZ*`!dM1p1gaS+C!`*sa<;30(T8kwh-4A|FxVAu^Yjp z$?1OFA9>85U`CiXoe>=Fcl01uS%P$BQxlmL@Qv{7_vO@Sr~4sfCdK-zQ<7c{g{!Wr zYDTn@M`LdNpx1l2zC1NIm#jK$l26aDu2%k=(sx(abD&kX`?&e99#%Px{cMo_x=FsIq(#f|Bvr$qt$kUs{RZ{6vZjp680#w^a+wn%sxdeFDr~;5dgsaG(J4Xw0RnZMQA1YnIPK=kB z=rPIERNUecG-@UOc*4@SRnkDR+VbXO8#61-4Mvd#z82dO^_Ob1jvo_lvA0Or9~C@M z`!Kgm{XtU5sKl#_y$Cmw*G&Kga? zB1*!1M6q0=5G#8_D!B5tDB%G})i2G&U0XP-7d?8cf0V8^gq`vT*M`O29khK<@J?-) z4wANtK6ChmkN+Ls=EKLSqOHSK8~w-%p3h5zhcgxwA<8AY72xzicq2Ht*a4)FEvs>A zYEt4;P|6XbFvmpy`ktVjqE}%Fa1} zNJ>P2WiL5CjsIbW3P0L^OulX()?^66v`V-5Cfb$w>h>=0iM`HN&|u5Rcw> zWZsk??@)Y<^N!M%uol0sAljiu7)pPE_PMh{>=)3kqR6rTJl0X!OF8BONry6AD5?e0 zF$>PcB<)xyB%mml$UjsbKqUe2kFmzNn548#JaG6(0Yw17CP;yLyUyD>FAhFoQp}S z=YhqQ!%lPxQb8gybI9_*1}x`dk}8ZBI;HQ;M>t5wz~>O^PxLmPF#ME*qzPO(lq6*n zUO7&@y7bRd8Wc42?i7$E5abwuIqaX6c5q|ARz#f1J)p5A22Xv7sc7M z5Osq$0REUQ?C{aYZInF$gO6meDaYp;>cwUX5rLmtuy=-)AVeK>^fVAL1PKLuTwEvx zgb-f`f-3^%7i>rI2!uX7rimy5<$-M^>w~N{WaJ5k!zgu#ChdLY2O@5a;ve5&E3i=}L5m WAd)B+SujzszZ`7Lw`2b 0.05 * image.max()) & (x >= x_c) + x0, y0 = (np.sum(c[iso] * image[iso]) / image[iso].sum() for c in (x, y)) + assert np.hypot(x0 - x_c, y0 - y_c) > 1.0 + + hlr = sigma * np.sqrt(2 * np.log(2)) + xw, yw, flags = wp.windowed_positions(image, [x0], [y0], [hlr]) + assert flags[0] == 0 + assert abs(xw[0] - x_c) < 1e-5 and abs(yw[0] - y_c) < 1e-5 + + +def test_neighbour_footprint_is_masked_by_its_mirror(): + """A neighbour's footprint does not pull the centroid. + + A neighbour ten times brighter, 8 px away, captures the window when + nothing masks it. With its footprint in the map, those pixels take the + galaxy's mirror image and the centroid stays within 0.02 px of the + isolated galaxy's (the rest is the neighbour's wings outside its + footprint). + """ + x_c, y_c, sigma = 30.0, 30.6, 2.0 + galaxy = _gaussian((61, 61), x_c, y_c, sigma) + neighbour = _gaussian((61, 61), x_c + 8.0, y_c + 1.0, 1.5, flux=20000.0) + y, x = np.mgrid[1:62, 1:62] + seg = np.where(np.hypot(x - x_c - 8.0, y - y_c - 1.0) < 5.5, 2, 0) + seg[np.hypot(x - x_c, y - y_c) < 3.0] = 1 + hlr = sigma * np.sqrt(2 * np.log(2)) + start = ([x_c + 0.5], [y_c], [hlr]) + + alone = wp.windowed_positions(galaxy, *start) + masked = wp.windowed_positions(galaxy + neighbour, *start, seg=seg, + number=[1]) + unmasked = wp.windowed_positions(galaxy + neighbour, *start) + assert np.hypot(masked[0][0] - alone[0][0], + masked[1][0] - alone[1][0]) < 0.02 + assert unmasked[0][0] - alone[0][0] > 1.0 + + +def test_unmeasurable_and_negative_objects_keep_the_start(): + """Off-image, NaN and negative-flux objects keep the input position.""" + image = _gaussian((41, 41), 20.0, 20.0, 2.0) + x0 = np.array([20.2, 60.0, np.nan]) + y0 = np.array([20.1, 20.0, 20.0]) + xw, yw, flags = wp.windowed_positions(image, x0, y0, [2.35, 2.35, 2.35]) + assert flags[0] == 0 + npt_equal = np.testing.assert_array_equal + npt_equal(flags[1:], wp.FLAGS_WIN_UNMEASURED) + npt_equal(xw[1:], x0[1:]) + + xw, yw, flags = wp.windowed_positions(-image, [20.2], [20.1], [2.35]) + assert flags[0] & wp.FLAGS_WIN_NEGATIVE_FLUX + assert (xw[0], yw[0]) == (20.2, 20.1) + + +def _fixture_objects(): + with np.load(FIXTURE) as data: + names = sorted({k.rsplit("_", 1)[0] for k in data.files + if k.endswith("_image")}) + return {n: {k[len(n) + 1:]: data[k] for k in data.files + if k.startswith(n + "_")} for n in names} + + +@pytest.mark.parametrize("name", sorted(_fixture_objects())) +def test_matches_sextractor_on_real_objects(name): + """SExtractor 2.25's XWIN_IMAGE and FLAGS_WIN, object by object. + + The objects are a bright and a faint isolated galaxy, two whose window + reaches a neighbour's footprint (masked_neighbour*: unmasked, they are + 0.07 and 0.6 px off), one SExtractor falls back to the barycentre for + (FLAGS_WIN 1) and one that has not converged after 16 steps. A fallback + agrees only to the float32 rounding of the catalogue's X_IMAGE. + """ + obj = _fixture_objects()[name] + x0, y0, fr = obj["start"] + xw, yw, flags = wp.windowed_positions( + obj["image"], [x0], [y0], [fr], seg=obj["seg"], number=[1], + cxx=obj["ellipse"][:1], cyy=obj["ellipse"][1:2], + cxy=obj["ellipse"][2:], + ) + expected = int(obj["flags_win"]) + assert flags[0] & 15 == expected + tol = 1e-3 if expected else 1e-5 + assert abs(xw[0] - obj["expected"][0]) < tol + assert abs(yw[0] - obj["expected"][1]) < tol + if name == "not_converged": + assert flags[0] & wp.FLAGS_WIN_NOT_CONVERGED + + +def test_background_recovers_a_smooth_sky(): + """A tilted sky plus sources comes back to well under the noise.""" + rng = np.random.default_rng(3) + ny, nx = 1200, 1500 + y, x = np.mgrid[0:ny, 0:nx] + sky = 5.0 + 1e-3 * x - 5e-4 * y + image = sky + rng.normal(0, 1.0, (ny, nx)) + for _ in range(100): + image += _gaussian((ny, nx), *rng.uniform(1, [nx, ny]), 2.0, + flux=rng.uniform(50, 500)) + back = wp.sextractor_background(image, back_size=256, filter_size=3) + assert np.abs(back - sky)[100:-100, 100:-100].max() < 0.1 diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini index ab9d04742..19e9d9d87 100644 --- a/workflow/config/cfis/config_tile_Uc.ini +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -88,8 +88,10 @@ SEGMENTATION = True # @sc [decision:detection.epoch_membership_ccd_bounds] MAKE_POST_PROCESS = True -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = ALPHA_J2000,DELTA_J2000 +# World coordinate keywords: the windowed position read_ext_sexcat measures. +# Format: KEY_X,KEY_Y +# @sc [decision:preparation.object_position_columns] +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD # Number of pixels in x,y of a CCD. Format: Nx,Ny # @sc [decision:detection.epoch_membership_ccd_bounds] diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index cc5feaabf..93a0c5f7a 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -2,6 +2,10 @@ # @sc [decision:preparation.object_position_columns] XWIN_WORLD YWIN_WORLD +# FLAGS_WIN: how the windowed position was obtained. 1-8 are SExtractor's +# reasons to fall back to the isophotal barycentre; under unions_catalogue +# also 16 (not measurable, barycentre kept) and 32 (not converged in 16 steps). +FLAGS_WIN # tile ID, for plot of tile-dependent additive bias. # Can maybe be removed. diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 9db095b66..96f806d05 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -15,6 +15,10 @@ # coordinates XWIN_WORLD YWIN_WORLD +# FLAGS_WIN: how the windowed position was obtained. 1-8 are SExtractor's +# reasons to fall back to the isophotal barycentre; under unions_catalogue +# also 16 (not measurable, barycentre kept) and 32 (not converged in 16 steps). +FLAGS_WIN # tile ID, for plot of tile-dependent additive bias. TILE_ID From da146762e684f86d98ea2c7bd7d6b915794a836e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 06:21:49 +0200 Subject: [PATCH 27/44] Windowed centroid: mirror no-data pixels, guard NaN and wild FLUX_RADIUS Review of #933 and the 202.301 run on Nibi: - Pixels that are 0 or not finite hold no data in a tile. They were left out of the background but read as signal in the window. They now take their mirror image through the centre, as SExtractor's MASK_TYPE CORRECT treats pixels under the weight threshold, and a mirror that is itself invalid counts as 0. - Non-finite pixels never enter the background or the sums: samples outside the aperture are zeroed before the product, and a non-finite sum gives FLAGS_WIN 16 at the barycentre. - The second moments are normalised as SExtractor's are after its loop, so a negative source is FLAGS_WIN 4, not 6. - FLUX_RADIUS <= 0 or NaN (DR6 carries values down to -1.2e6) is an empty window, FLAGS_WIN 4; windows wider than MAX_HALF = 256 px are FLAGS_WIN 16; and the working box never exceeds the image. The -12402 radius on 202.301 had built an 84k-px box and been killed at 16 GB. On 186.307 the agreement with SExtractor is unchanged (2.8% of objects off by more than 0.01 px on the DR5 tile, 3.4% on the DR6 path). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NsPvmF43rzKrb5eYPNR3Ki --- astra.yaml | 9 +- .../read_ext_sexcat.py | 9 +- .../windowed_position.py | 139 +++++++++++------- tests/module/test_read_ext_sexcat.py | 29 ++++ tests/module/test_windowed_position.py | 128 ++++++++++++++++ 5 files changed, 259 insertions(+), 55 deletions(-) diff --git a/astra.yaml b/astra.yaml index f664b0863..ef0af62da 100644 --- a/astra.yaml +++ b/astra.yaml @@ -803,7 +803,14 @@ analyses: SExtractor's own XWIN_IMAGE to 0.01 px for 97% of objects, the rest being blends whose SExtractor masking the segmentation map does not fully record; the barycentre it replaces is off by 0.16 px at the - median and 1.4 px at the 99th percentile. Where SExtractor would + median and 1.4 px at the 99th percentile. On tile 202.301 the rest + follows DR6's FLUX_RADIUS, which sizes the window: where it agrees + with ShapePipe's SExtractor to 1e-3 px the match is 4e-5 px at the + median and 2e-4 px at p90. Tile pixels that are 0 or not finite + carry no data and are mirrored like neighbours; the weight map is + not read, so SExtractor's MAP_WEIGHT thresholds and INTERP_TYPE ALL + interpolation are not emulated. A FLUX_RADIUS that is not positive + is an empty window (FLAGS_WIN 4). Where SExtractor would fall back to the barycentre (FLAGS_WIN 1, 2, 4, 8), or nothing can be measured (16), the position is the barycentre; FLAGS_WIN, in the final catalogue, records it, and 32 marks a centroid still moving diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py index fa698d312..182707b53 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py @@ -211,8 +211,9 @@ def measure_windowed_positions(cat_data, image_data, img_header, seg=None): ``Y_IMAGE``) and half-light radius (``FLUX_RADIUS``) start and size SExtractor's windowed centroid, measured on the tile image less a SExtractor-like background map (``default_tile.sex``: ``BACK_SIZE`` - 512, ``BACK_FILTERSIZE`` 9). Pixels exactly 0 carry no data in a - MegaPipe tile and are left out of the background. Given the relabelled + 512, ``BACK_FILTERSIZE`` 9). Pixels exactly 0 or not finite carry no + data in a MegaPipe tile: they are left out of the background and + replaced by their mirror image in the window. Given the relabelled segmentation map, neighbours are masked as SExtractor's ``MASK_TYPE CORRECT`` masks them. Where the window fails, the position stays the barycentre and ``FLAGS_WIN`` says why @@ -254,12 +255,14 @@ def measure_windowed_positions(cat_data, image_data, img_header, seg=None): cat_data["A_WORLD"], cat_data["B_WORLD"], cat_data["THETA_J2000"], wcs, x, y, ) + has_data = (image_data != 0) & np.isfinite(image_data) measured = image_data - wp.sextractor_background( - image_data, good=image_data != 0 + image_data, good=has_data ) xwin, ywin, flags = wp.windowed_positions( measured, x, y, cat_data["FLUX_RADIUS"], seg=seg, number=np.asarray(cat_data["NUMBER"]), cxx=cxx, cyy=cyy, cxy=cxy, + valid=has_data, ) ra, dec = wcs.all_pix2world(xwin, ywin, 1) return [ diff --git a/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py b/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py index 4c7659063..4dbade857 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py @@ -31,7 +31,7 @@ FLAGS_WIN_NEGATIVE_MOMENT = 2 # a windowed second moment is negative FLAGS_WIN_NEGATIVE_FLUX = 4 # windowed flux is not positive FLAGS_WIN_WANDERED = 8 # centroid left the isophotal 1-sigma ellipse -FLAGS_WIN_UNMEASURED = 16 # no measurement: barycentre not finite or off image +FLAGS_WIN_UNMEASURED = 16 # not measured: bad start, huge window, NaN sums FLAGS_WIN_NOT_CONVERGED = 32 # still moving after 16 steps; position kept FLAGS_WIN_FALLBACK = ( FLAGS_WIN_SINGULAR @@ -44,6 +44,11 @@ # Pixels per object and chunk, bounding the working arrays (~100 MB). _CHUNK_PIXELS = 2_000_000 +# Largest stamp half-width, in pixels: a window radius of 256 px, which a +# FLUX_RADIUS of about 75 px reaches. Larger windows are not measured +# (FLAGS_WIN 16), so a wild FLUX_RADIUS cannot allocate without bound. +MAX_HALF = 256 + def _natural_spline(nodes, at, axis): """Natural cubic spline through equally spaced nodes 0, 1, ..., n - 1. @@ -101,7 +106,7 @@ def sextractor_background(image, back_size=512, filter_size=9, good=None): ``BACK_FILTERSIZE``, the median-filter side in meshes good : numpy.ndarray, optional Boolean map of the pixels to use (SExtractor: weight above - threshold); default all finite pixels + threshold); non-finite pixels are never used Returns ------- @@ -112,8 +117,7 @@ def sextractor_background(image, back_size=512, filter_size=9, good=None): """ ny, nx = image.shape nbx, nby = (nx - 1) // back_size + 1, (ny - 1) // back_size + 1 - if good is None: - good = np.isfinite(image) + good = np.isfinite(image) if good is None else good & np.isfinite(image) back = np.full((nby, nbx), np.nan) for j in range(nby): for i in range(nbx): @@ -181,18 +185,20 @@ def _blanked(label, own): return (label != 0) & (label != own) -def _iterate(image, mx, my, sig, seg=None, own=None, +def _iterate(image, valid, mx, my, sig, seg=None, own=None, n_iter_max=WINPOS_NITERMAX): """Run ``compute_winpos``'s loop for objects sharing one stamp size. ``mx``, ``my`` are 0-based starting positions and are not modified. - With ``seg``, pixels :func:`_blanked` for the object take the value - of their mirror image through the current centre, or 0 where the - mirror is off the image or blanked too: SExtractor's ``MASK_TYPE - CORRECT``. Returns the final - 0-based positions, the windowed flux ``tv``, the windowed second - moments of the last step, and whether each object stopped on the step - criterion. + Pixels not ``valid`` (SExtractor: weight below threshold) and, with + ``seg``, pixels :func:`_blanked` for the object take the value of + their mirror image through the current centre, or 0 where the mirror + is off the image, not valid or blanked: SExtractor's ``MASK_TYPE + CORRECT``. Returns the final 0-based positions, the windowed flux + ``tv``, the windowed second moments as SExtractor normalises them + after its last step, whether each object stopped on the step + criterion, and whether a sum went non-finite (the object is then + dropped at its last finite position). """ ny, nx = image.shape n = len(mx) @@ -202,12 +208,13 @@ def _iterate(image, mx, my, sig, seg=None, own=None, raper2 = raper * raper rintlim2 = np.clip(raper - 0.75, 0, None) ** 2 rextlim2 = (raper + 0.75) ** 2 - half = int(np.ceil(raper.max())) + 2 + half = min(int(np.ceil(raper.max())) + 2, max(nx, ny)) offs = np.arange(-half, half + 1) tv = np.zeros(n) mom = np.zeros((3, n)) converged = np.zeros(n, bool) + failed = np.zeros(n, bool) active = np.arange(n) for _ in range(n_iter_max): if active.size == 0: @@ -237,55 +244,73 @@ def _iterate(image, mx, my, sig, seg=None, own=None, area *= in_y[:, :, None] & in_x[:, None, :] cy = np.clip(ys, 0, ny - 1)[:, :, None] cx = np.clip(xs, 0, nx - 1)[:, None, :] - pix = image[cy, cx] + pix = image[cy, cx].astype(np.float64) + mask = ~valid[cy, cx] if seg is not None: - lab = seg[cy, cx] - blank = _blanked(lab, own[active, None, None]) & (area > 0) - if blank.any(): - k, iy, ix = np.nonzero(blank) - # (int)(2 m + 0.49999 - x), truncating as C does. - x2 = np.trunc(2 * amx[k] + 0.49999 - xs[k, ix]) - y2 = np.trunc(2 * amy[k] + 0.49999 - ys[k, iy]) - x2, y2 = x2.astype(np.int64), y2.astype(np.int64) - inside = (x2 >= 0) & (x2 < nx) & (y2 >= 0) & (y2 < ny) - x2c, y2c = np.clip(x2, 0, nx - 1), np.clip(y2, 0, ny - 1) - mlab = seg[y2c, x2c] - usable = inside & ~_blanked(mlab, own[active][k]) - pix = pix.astype(np.float64) - pix[k, iy, ix] = np.where(usable, image[y2c, x2c], 0.0) + mask |= _blanked(seg[cy, cx], own[active, None, None]) + mask &= area > 0 + if mask.any(): + k, iy, ix = np.nonzero(mask) + # (int)(2 m + 0.49999 - x), truncating as C does. + x2 = np.trunc(2 * amx[k] + 0.49999 - xs[k, ix]).astype(np.int64) + y2 = np.trunc(2 * amy[k] + 0.49999 - ys[k, iy]).astype(np.int64) + inside = (x2 >= 0) & (x2 < nx) & (y2 >= 0) & (y2 < ny) + x2, y2 = np.clip(x2, 0, nx - 1), np.clip(y2, 0, ny - 1) + usable = inside & valid[y2, x2] + if seg is not None: + usable &= ~_blanked(seg[y2, x2], own[active][k]) + pix[k, iy, ix] = np.where(usable, image[y2, x2], 0.0) + # Samples outside the aperture are 0, whatever the image holds. + pix = np.where(area > 0, pix, 0.0) locpix = area * np.exp(-r2 * invtwosig2[active, None, None]) * pix atv = locpix.sum((1, 2)) - pos = atv > 0 - safe = np.where(pos, atv, 1.0) - dxpos = (locpix * dx).sum((1, 2)) / safe - dypos = (locpix * dy).sum((1, 2)) / safe + sx = (locpix * dx).sum((1, 2)) + sy = (locpix * dy).sum((1, 2)) + sxx = (locpix * dx * dx).sum((1, 2)) + syy = (locpix * dy * dy).sum((1, 2)) + sxy = (locpix * dx * dy).sum((1, 2)) + finite = np.isfinite(atv) & np.isfinite(sx) & np.isfinite(sy) + failed[active[~finite]] = True + pos = finite & (atv > 0) + # SExtractor divides the offsets by tv only when tv > 0, and + # normalises the moments by tv after the loop either way. + with np.errstate(divide="ignore", invalid="ignore"): + dxpos = np.where(pos, sx / np.where(pos, atv, 1.0), sx) + dypos = np.where(pos, sy / np.where(pos, atv, 1.0), sy) + mom[0, active] = sxx / atv - dxpos**2 + mom[1, active] = syy / atv - dypos**2 + mom[2, active] = sxy / atv - dxpos * dypos tv[active] = atv - mom[0, active] = (locpix * dx * dx).sum((1, 2)) / safe - dxpos**2 - mom[1, active] = (locpix * dy * dy).sum((1, 2)) / safe - dypos**2 - mom[2, active] = (locpix * dx * dy).sum((1, 2)) / safe - dxpos * dypos mx[active] += np.where(pos, dxpos * WINPOS_FAC, 0.0) my[active] += np.where(pos, dypos * WINPOS_FAC, 0.0) done = pos & (dxpos**2 + dypos**2 < WINPOS_STEPMIN**2) converged[active[done]] = True active = active[pos & ~done] - return mx, my, tv, mom, converged + return mx, my, tv, mom, converged, failed def windowed_positions(image, x_image, y_image, flux_radius, seg=None, - number=None, cxx=None, cyy=None, cxy=None): + number=None, cxx=None, cyy=None, cxy=None, + valid=None): """SExtractor's windowed centroid, started from given positions. For each object, a Gaussian window of sigma - ``2 * hl / 2.35`` (``hl`` = ``|flux_radius|``, at least 0.5 pixel) + ``2 * hl / 2.35`` (``hl`` = ``flux_radius``, at least 0.5 pixel) weights the image within ``WINPOS_NSIG`` sigmas, the aperture edge subsampled 11 x 11; the centre moves by ``WINPOS_FAC`` times the weighted mean offset, up to ``WINPOS_NITERMAX`` times, until that - offset is below ``WINPOS_STEPMIN``. Given the segmentation map, - neighbours' footprints take the mirror image of the object's side, as - SExtractor's ``MASK_TYPE CORRECT`` does. Where SExtractor would revert - to the isophotal position (``FLAGS_WIN`` 1, 2, 4, 8) this returns the - input position, and so it does where there is nothing to measure (16). + offset is below ``WINPOS_STEPMIN``. Invalid pixels and, given the + segmentation map, neighbours' footprints take the mirror image of the + object's side, as SExtractor's ``MASK_TYPE CORRECT`` does. Where + SExtractor would revert to the isophotal position (``FLAGS_WIN`` 1, 2, + 4, 8) this returns the input position, and so it does where there is + nothing to measure, the window is wider than ``MAX_HALF`` or a sum is + not finite (16). A ``flux_radius`` that is not positive or not finite + gives an empty aperture, so zero windowed flux (4). The tile weight map + is not used: SExtractor's ``WEIGHT_TYPE MAP_WEIGHT`` thresholds and + ``INTERP_TYPE ALL`` interpolation of zero-weight pixels are not + emulated; pixels outside ``valid`` are mirrored instead. A centroid still moving after the last step keeps its position, as SExtractor's does, and is flagged 32. @@ -306,6 +331,9 @@ def windowed_positions(image, x_image, y_image, flux_radius, seg=None, cxx, cyy, cxy : array_like, optional Isophotal ellipse (``CXX_IMAGE`` ...) for the flag-8 test; without them bit 8 is never set + valid : numpy.ndarray, optional + Boolean map of the pixels holding data (SExtractor: weight above + threshold); non-finite pixels are never valid Returns ------- @@ -326,30 +354,39 @@ def windowed_positions(image, x_image, y_image, flux_radius, seg=None, flags = np.zeros(n, np.int16) xwin, ywin = x0.copy(), y0.copy() - ok = np.isfinite(x0) & np.isfinite(y0) & np.isfinite(fr) + ok = np.isfinite(x0) & np.isfinite(y0) ok &= (x0 >= 0.5) & (x0 < nx + 0.5) & (y0 >= 0.5) & (y0 < ny + 0.5) - flags[~ok] |= FLAGS_WIN_UNMEASURED - sig = np.maximum(np.abs(np.where(ok, fr, 1.0)), GROWTH_MINHLRAD) * 2 / 2.35 + empty = ok & ~(fr > 0) + flags[empty] |= FLAGS_WIN_NEGATIVE_FLUX + sig = np.maximum(np.where(fr > 0, fr, 1.0), GROWTH_MINHLRAD) * 2 / 2.35 half = np.ceil(WINPOS_NSIG * sig).astype(np.int64) + 2 + ok &= ~empty & (half <= MAX_HALF) + flags[~ok & ~empty] |= FLAGS_WIN_UNMEASURED + # A box wider than the image holds nothing more than the image. + half = np.minimum(half, max(nx, ny)) + finite = np.isfinite(image) + valid = finite if valid is None else valid & finite own = None if seg is None else np.asarray(number) tv = np.zeros(n) mom = np.zeros((3, n)) + failed = np.zeros(n, bool) for h in np.unique(half[ok]): members = np.flatnonzero(ok & (half == h)) size = (2 * h + 1) ** 2 for c0 in range(0, len(members), max(1, _CHUNK_PIXELS // size)): idx = members[c0:c0 + max(1, _CHUNK_PIXELS // size)] - mx, my, tv[idx], mom[:, idx], conv = _iterate( - image, x0[idx] - 1.0, y0[idx] - 1.0, sig[idx], seg, + mx, my, tv[idx], mom[:, idx], conv, failed[idx] = _iterate( + image, valid, x0[idx] - 1.0, y0[idx] - 1.0, sig[idx], seg, None if own is None else own[idx], ) xwin[idx], ywin[idx] = mx + 1.0, my + 1.0 - stopped_on_flux = tv[idx] <= 0 - flags[idx[~conv & ~stopped_on_flux]] |= FLAGS_WIN_NOT_CONVERGED + stopped = conv | (tv[idx] <= 0) | failed[idx] + flags[idx[~stopped]] |= FLAGS_WIN_NOT_CONVERGED + flags[failed] |= FLAGS_WIN_UNMEASURED mx2, my2, mxy = mom - measured = ok + measured = ok & ~failed flags[measured & (mx2 * my2 - mxy * mxy < 0)] |= FLAGS_WIN_SINGULAR flags[measured & ((mx2 < 0) | (my2 < 0))] |= FLAGS_WIN_NEGATIVE_MOMENT flags[measured & (tv <= 0)] |= FLAGS_WIN_NEGATIVE_FLUX diff --git a/tests/module/test_read_ext_sexcat.py b/tests/module/test_read_ext_sexcat.py index b77adf306..d06b42f01 100644 --- a/tests/module/test_read_ext_sexcat.py +++ b/tests/module/test_read_ext_sexcat.py @@ -151,6 +151,35 @@ def test_converter_measures_windowed_centroids(tmp_path): npt.assert_allclose(data["X_IMAGE"], [t[1] + 0.6 for t in truth]) +@pytest.mark.decision("preparation.object_position_columns") +def test_converter_mirrors_no_data_pixels(tmp_path): + """Zero (no-data) tile columns through a galaxy do not move XWIN. + + A Gaussian at (31, 31) on a sky of 3, with columns 33-35 set to 0 as a + tile gap is: the gap is left out of the background and mirrored in the + window, so a centroid started at the centre stays there (read as + signal, the gap drags it 0.6 px). + """ + y, x = np.mgrid[1:62, 1:62] + image = 3.0 + 500 * np.exp(-((x - 31.0) ** 2 + (y - 31.0) ** 2) / 8.0) + image[:, 32:35] = 0.0 + img = tmp_path / "CFIS_image-301-279.fits" + hdu = fits.PrimaryHDU(image.astype(np.float32)) + hdu.header.update(TILE_WCS) + hdu.writeto(img) + cat = tmp_path / "CFIS_cat-301-279.cat" + hlr = 2.0 * np.sqrt(2 * np.log(2)) + cat.write_text("# 1 NUMBER\n# 2 X_IMAGE\n# 3 Y_IMAGE\n" + f"# 4 FLUX_RADIUS\n1 31.0 31.0 {hlr}\n") + out = tmp_path / "sexcat-301-279.fits" + rs.make_ldac_from_ascii(str(cat), str(img), str(out), stamp_size=5) + with fits.open(out) as hdul: + data = hdul["LDAC_OBJECTS"].data + assert data["FLAGS_WIN"][0] == 0 + npt.assert_allclose(data["XWIN_IMAGE"], 31.0, atol=1e-3) + npt.assert_allclose(data["YWIN_IMAGE"], 31.0, atol=1e-3) + + def test_converter_needs_flux_radius(tmp_path): """Without FLUX_RADIUS there is no window, and the converter stops.""" cat = tmp_path / "CFIS_cat-301-279.cat" diff --git a/tests/module/test_windowed_position.py b/tests/module/test_windowed_position.py index 41d7eddda..b67520227 100644 --- a/tests/module/test_windowed_position.py +++ b/tests/module/test_windowed_position.py @@ -95,6 +95,134 @@ def test_unmeasurable_and_negative_objects_keep_the_start(): assert (xw[0], yw[0]) == (20.2, 20.1) +def test_negative_flux_flag_is_sextractors(): + """A centred negative source is FLAGS_WIN 4 exactly, as in SExtractor. + + SExtractor divides the (negative) second-moment sums by the negative + flux, so the moments come out positive and only bit 4 is set. + """ + image = -_gaussian((41, 41), 20.0, 20.0, 2.0) + _, _, flags = wp.windowed_positions(image, [20.0], [20.0], [2.35]) + assert flags[0] == wp.FLAGS_WIN_NEGATIVE_FLUX + + +HLR = 2.0 * np.sqrt(2 * np.log(2)) + + +def test_invalid_stripe_is_mirrored(): + """A no-data stripe through the window takes the mirror image. + + A Gaussian at (31, 31) with columns 33-35 holding no data: what the + stripe holds is ignored, and a centroid at the centre stays there. Read + as signal, the zero stripe drags it 0.6 px left. (From other starts + SExtractor's integer mirror index can settle up to half a pixel off, at + 31.44 from (31.3, 30.8); this is SExtractor's behaviour too.) + """ + image = _gaussian((61, 61), 31.0, 31.0, 2.0) + valid = np.ones(image.shape, bool) + valid[:, 32:35] = False + zero = np.where(valid, image, 0.0) + garbage = np.where(valid, image, 1e4) + + xw, yw, flags = wp.windowed_positions(zero, [31.0], [31.0], [HLR], + valid=valid) + assert flags[0] == 0 + assert abs(xw[0] - 31.0) < 1e-6 and abs(yw[0] - 31.0) < 1e-6 + for start in (([31.0], [31.0]), ([31.3], [30.8])): + a = wp.windowed_positions(zero, *start, [HLR], valid=valid) + b = wp.windowed_positions(garbage, *start, [HLR], valid=valid) + np.testing.assert_array_equal(a[0], b[0]) + np.testing.assert_array_equal(a[1], b[1]) + xw, _, _ = wp.windowed_positions(zero, [31.0], [31.0], [HLR]) + assert xw[0] < 30.8 + + +def test_invalid_and_off_image_mirrors_count_as_zero(): + """A mirror off the image, or itself invalid, contributes 0. + + Invalid pixels hold garbage; the result equals that of the same image + with those pixels set to 0 and all valid, near the left edge (mirrors + off the image) and with both a stripe and its mirror invalid. + """ + rng = np.random.default_rng(5) + for x_c, cols in ((4.0, np.r_[8:11]), (31.0, np.r_[26:29, 32:35])): + image = _gaussian((61, 61), x_c, 31.0, 2.0) + valid = np.ones(image.shape, bool) + valid[:, cols] = False + zeroed = np.where(valid, image, 0.0) + garbage = np.where(valid, image, rng.uniform(1e3, 1e4, image.shape)) + start = ([x_c + 0.1], [31.2], [HLR]) + got = wp.windowed_positions(garbage, *start, valid=valid) + want = wp.windowed_positions(zeroed, *start) + np.testing.assert_allclose(got[:2], want[:2], rtol=0, atol=1e-9) + assert got[2][0] == want[2][0] + + +def test_nan_outside_the_aperture_is_ignored(): + """A NaN that enters the stamp square after the first step changes nothing. + + From (30.2, 30.2) the square first spans 0-based columns 18-40; once + the centre reaches (31, 31) it spans 19-41, and a NaN at column 41, + outside the aperture, must not reach the sums. + """ + image = _gaussian((61, 61), 31.0, 31.0, 2.0) + clean = wp.windowed_positions(image, [30.2], [30.2], [HLR]) + image[30, 41] = np.nan + got = wp.windowed_positions(image, [30.2], [30.2], [HLR]) + assert got[2][0] == 0 + np.testing.assert_allclose(got[:2], clean[:2], rtol=0, atol=1e-12) + + +def test_nan_inside_the_aperture_is_mirrored(): + """A NaN pixel on the galaxy is invalid and takes its mirror image.""" + image = _gaussian((61, 61), 31.0, 31.0, 2.0) + image[31, 32] = np.nan + xw, yw, flags = wp.windowed_positions(image, [30.6], [31.3], [HLR]) + assert flags[0] == 0 + assert abs(xw[0] - 31.0) < 1e-3 and abs(yw[0] - 31.0) < 1e-3 + + +def test_wild_flux_radius_is_flagged_without_allocating(): + """FLUX_RADIUS <= 0 or NaN is an empty aperture; a huge one is skipped. + + DR6 carries FLUX_RADIUS down to -1.2e6; taken as a size, it built an + 84k-px box. Non-positive and NaN radii give FLAGS_WIN 4, radii whose + window exceeds MAX_HALF give 16, both at the barycentre, and the + measurable object alongside is unaffected. + """ + import tracemalloc + + image = _gaussian((101, 101), 51.0, 51.0, 2.0) + fr = [-12402.6, 0.0, np.nan, 1.18e6, HLR] + x0 = [50.5, 50.5, 50.5, 50.5, 50.6] + tracemalloc.start() + xw, yw, flags = wp.windowed_positions(image, x0, [51.0] * 5, fr) + peak = tracemalloc.get_traced_memory()[1] + tracemalloc.stop() + assert peak < 200e6 + np.testing.assert_array_equal( + flags, [4, 4, 4, wp.FLAGS_WIN_UNMEASURED, 0]) + np.testing.assert_array_equal(xw[:4], x0[:4]) + assert abs(xw[4] - 51.0) < 1e-5 + + +def test_box_is_clamped_to_a_small_image(): + """A window wider than the image measures the image, not a huge box.""" + image = _gaussian((9, 9), 5.0, 5.0, 1.0) + xw, yw, flags = wp.windowed_positions(image, [5.2], [4.9], [60.0]) + assert flags[0] & wp.FLAGS_WIN_UNMEASURED == 0 + assert np.isfinite(xw[0]) + + +def test_background_ignores_nan_pixels(): + """A NaN in a flat sky-3 mesh leaves the background at 3.""" + image = np.full((200, 200), 3.0) + image[100, 100] = np.nan + back = wp.sextractor_background(image, back_size=256, filter_size=3, + good=image != 0) + np.testing.assert_allclose(back, 3.0, atol=1e-6) + + def _fixture_objects(): with np.load(FIXTURE) as data: names = sorted({k.rsplit("_", 1)[0] for k in data.files From e167e0f6eb14af430d8765eb7db5958ed706ad3c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 03:37:26 +0200 Subject: [PATCH 28/44] Revert the windowed-centroid port The port re-implemented SExtractor's windowed centroid in numpy for UNIONS-catalogue tiles. Running SExtractor itself on the DR6 image and joining its rows to the DR6 catalogue replaces it (next commits). Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 65 +-- .../read_ext_sexcat.py | 121 +---- .../windowed_position.py | 442 ------------------ .../modules/read_ext_sexcat_runner.py | 7 +- tests/module/data/xwin_sextractor_186307.npz | Bin 23951 -> 0 bytes tests/module/test_read_ext_sexcat.py | 127 +---- tests/module/test_windowed_position.py | 271 ----------- workflow/config/cfis/config_tile_Uc.ini | 6 +- workflow/config/cfis/final_cat.param | 4 - .../config/cfis_image_sims/final_cat.param | 4 - 10 files changed, 49 insertions(+), 998 deletions(-) delete mode 100644 src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py delete mode 100644 tests/module/data/xwin_sextractor_186307.npz delete mode 100644 tests/module/test_windowed_position.py diff --git a/astra.yaml b/astra.yaml index ef0af62da..24dd350d3 100644 --- a/astra.yaml +++ b/astra.yaml @@ -440,12 +440,8 @@ analyses: with a finer mesh and a GLOBAL one. ngmix subtracts the exposure BACKGROUND map from each epoch and weights its pixels by BACKGROUND_RMS (shape_measurement.galaxy_pixel_weights). The header - background path is off. Under tile_detection = unions_catalogue, - read_ext_sexcat re-estimates the tile background with the same mesh - and filter (clipped mode per mesh, median filter, bicubic spline) to - measure windowed centroids (preparation.object_position_columns). - Residual sky offsets propagate into thresholds, fluxes, completeness - and shapes. + background path is off. Residual sky offsets propagate into + thresholds, fluxes, completeness and shapes. Values: default_tile.sex#BACK_TYPE = AUTO; default_tile.sex#BACK_SIZE = 512; @@ -784,39 +780,13 @@ analyses: object_position_columns: label: Windowed centroids (XWIN/YWIN) define every position rationale: >- - PSF interpolation sites, tile and multi-epoch stamp centres, epoch - membership and the catalogue position all use SExtractor's windowed - centroid: in pixels for tile stamps and exposure-side PSF - validation, in world coordinates for tile-side PSF interpolation, - multi-epoch stamps, epoch membership and the catalogue. It is a - Gaussian-weighted centroid iterated from the isophotal barycentre, - window sigma 2 FLUX_RADIUS / 2.35, and it feeds the stamp OFFSET and - the ngmix centroid prior (shape_measurement.centroid_source). The - UNIONS per-tile catalogue (detection.tile_detection = - unions_catalogue) carries only the barycentre, so read_ext_sexcat - measures the windowed centroid itself at every catalogue object: a - numpy port of SExtractor 2.25's compute_winpos, started from X_IMAGE, - Y_IMAGE with the catalogue's FLUX_RADIUS, on the tile less a - SExtractor-like background (detection.background_model), with - neighbours' segmentation footprints replaced by their mirror image - as MASK_TYPE CORRECT does. On DR6 tile 186.307 it matches the tile - SExtractor's own XWIN_IMAGE to 0.01 px for 97% of objects, the rest - being blends whose SExtractor masking the segmentation map does not - fully record; the barycentre it replaces is off by 0.16 px at the - median and 1.4 px at the 99th percentile. On tile 202.301 the rest - follows DR6's FLUX_RADIUS, which sizes the window: where it agrees - with ShapePipe's SExtractor to 1e-3 px the match is 4e-5 px at the - median and 2e-4 px at p90. Tile pixels that are 0 or not finite - carry no data and are mirrored like neighbours; the weight map is - not read, so SExtractor's MAP_WEIGHT thresholds and INTERP_TYPE ALL - interpolation are not emulated. A FLUX_RADIUS that is not positive - is an empty window (FLAGS_WIN 4). Where SExtractor would - fall back to the barycentre (FLAGS_WIN 1, 2, 4, 8), or nothing can - be measured (16), the position is the barycentre; FLAGS_WIN, in the - final catalogue, records it, and 32 marks a centroid still moving - after 16 steps, kept as SExtractor keeps it. The tile VIGNET is - centred on the barycentre, as SExtractor's is. Image simulations - (sextractor) take the same quantity from SExtractor. + PSF interpolation sites, tile and multi-epoch stamp centres, and the + catalogue position all use SExtractor's windowed centroid: in pixels + for tile stamps and exposure-side PSF validation, in world + coordinates for tile-side PSF interpolation and multi-epoch stamps. + Windowed, isophotal and model centroids differ systematically for + blends and asymmetric galaxies, and the centroid feeds the position + seed and the centroid prior. Values: VIGNETMAKER_RUNNER_RUN_1.POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE; VIGNETMAKER_RUNNER_RUN_1.COORD = PIX; @@ -825,24 +795,11 @@ analyses: VIGNETMAKER_RUNNER_RUN_2.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD; VIGNETMAKER_RUNNER_RUN_2.COORD = SPHE; config_exp_psfex.ini#PSFEX_INTERP_RUNNER.POSITION_PARAMS = - XWIN_IMAGE,YWIN_IMAGE; - config_tile_Uc.ini#READ_EXT_SEXCAT_RUNNER.WORLD_POSITION = - XWIN_WORLD,YWIN_WORLD; - config_tile_Sx.ini#SEXTRACTOR_RUNNER.WORLD_POSITION = - XWIN_WORLD,YWIN_WORLD. + XWIN_IMAGE,YWIN_IMAGE. default: xwin_windowed options: xwin_windowed: - label: >- - Windowed centroids everywhere, measured by ShapePipe on - UNIONS-catalogue tiles - catalogue_barycentre: - label: The UNIONS catalogue's isophotal barycentre under the XWIN names - excluded: true - excluded_reason: >- - Image simulations measure windowed centroids, so data and - simulations would differ in the centroid the stamp OFFSET and - the ngmix centroid prior start from. + label: Windowed centroids everywhere stamp_positioning_and_padding: label: Nearest-pixel stamp extraction with zero padding rationale: >- diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py index 182707b53..ff4f58cc8 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py @@ -1,8 +1,7 @@ """READ EXTERNAL SEXCAT. Convert an ASCII SExtractor-format catalogue to FITS-LDAC for use in -ShapePipe tile processing, measuring the windowed positions the catalogue -lacks on the tile image. +ShapePipe tile processing. :Author: Martin Kilbinger @@ -11,9 +10,6 @@ import numpy as np from astropy.io import ascii as asc from astropy.io import fits -from astropy.wcs import WCS - -from shapepipe.modules.read_ext_sexcat_package import windowed_position as wp def _build_ldac_imhead(img_header): @@ -204,77 +200,7 @@ def _extract_vignets(image_data, x_pos, y_pos, stamp_size, seg=None, return vignets -def measure_windowed_positions(cat_data, image_data, img_header, seg=None): - """``XWIN_*``, ``YWIN_*`` and ``FLAGS_WIN`` for a catalogue without them. - - The catalogue is the sample; its isophotal barycentre (``X_IMAGE``, - ``Y_IMAGE``) and half-light radius (``FLUX_RADIUS``) start and size - SExtractor's windowed centroid, measured on the tile image less a - SExtractor-like background map (``default_tile.sex``: ``BACK_SIZE`` - 512, ``BACK_FILTERSIZE`` 9). Pixels exactly 0 or not finite carry no - data in a MegaPipe tile: they are left out of the background and - replaced by their mirror image in the window. Given the relabelled - segmentation map, neighbours are masked as SExtractor's ``MASK_TYPE - CORRECT`` masks them. Where the window fails, the position stays the - barycentre and ``FLAGS_WIN`` says why - (:mod:`windowed_position`). World positions come from the tile WCS, - 1-based as SExtractor's are. - - Parameters - ---------- - cat_data : astropy.table.Table - Catalogue with ``NUMBER``, ``X_IMAGE``, ``Y_IMAGE``, ``FLUX_RADIUS``, - and optionally ``A_WORLD``, ``B_WORLD``, ``THETA_J2000`` - image_data : numpy.ndarray - Tile image - img_header : astropy.io.fits.Header - Tile header, for the WCS - seg : numpy.ndarray, optional - Segmentation map relabelled to ``NUMBER`` (:func:`relabel_seg`) - - Returns - ------- - list of (str, numpy.ndarray, str) - ``XWIN_IMAGE``, ``YWIN_IMAGE``, ``XWIN_WORLD``, ``YWIN_WORLD``, - ``FLAGS_WIN`` columns with their FITS formats - - @sc [decision:preparation.object_position_columns] - """ - missing = {"X_IMAGE", "Y_IMAGE", "FLUX_RADIUS"} - set(cat_data.colnames) - if missing: - raise ValueError( - f"The catalogue lacks {sorted(missing)}, which the windowed" - + " centroid needs." - ) - x = np.asarray(cat_data["X_IMAGE"], dtype=np.float64) - y = np.asarray(cat_data["Y_IMAGE"], dtype=np.float64) - wcs = WCS(img_header) - cxx = cyy = cxy = None - if {"A_WORLD", "B_WORLD", "THETA_J2000"} <= set(cat_data.colnames): - cxx, cyy, cxy = wp.isophotal_ellipse( - cat_data["A_WORLD"], cat_data["B_WORLD"], cat_data["THETA_J2000"], - wcs, x, y, - ) - has_data = (image_data != 0) & np.isfinite(image_data) - measured = image_data - wp.sextractor_background( - image_data, good=has_data - ) - xwin, ywin, flags = wp.windowed_positions( - measured, x, y, cat_data["FLUX_RADIUS"], seg=seg, - number=np.asarray(cat_data["NUMBER"]), cxx=cxx, cyy=cyy, cxy=cxy, - valid=has_data, - ) - ra, dec = wcs.all_pix2world(xwin, ywin, 1) - return [ - ("XWIN_IMAGE", xwin, "D"), - ("YWIN_IMAGE", ywin, "D"), - ("XWIN_WORLD", ra, "D"), - ("YWIN_WORLD", dec, "D"), - ("FLAGS_WIN", flags, "I"), - ] - - -def _build_ldac_objects(cat_data, vignets, windowed=()): +def _build_ldac_objects(cat_data, vignets): """Build LDAC_OBJECTS extension from an astropy table. Parameters @@ -283,8 +209,6 @@ def _build_ldac_objects(cat_data, vignets, windowed=()): Catalogue data read from the ASCII SExtractor file vignets : numpy.ndarray Array of shape ``(n_obj, stamp_size, stamp_size)`` - windowed : sequence of (str, numpy.ndarray, str), optional - Extra ``(name, array, FITS format)`` columns, the windowed positions Returns ------- @@ -307,8 +231,23 @@ def _build_ldac_objects(cat_data, vignets, windowed=()): fmt = f"{arr.dtype.itemsize}A" fits_cols.append(fits.Column(name=colname, format=fmt, array=arr)) - for name, arr, fmt in windowed: - fits_cols.append(fits.Column(name=name, format=fmt, array=arr)) + _aliases = { + "XWIN_IMAGE": "X_IMAGE", + "YWIN_IMAGE": "Y_IMAGE", + "XWIN_WORLD": "ALPHA_J2000", + "YWIN_WORLD": "DELTA_J2000", + } + col_map = {c.name: c for c in fits_cols} + for alias, source in _aliases.items(): + if source in col_map and alias not in col_map: + src_col = col_map[source] + fits_cols.append( + fits.Column( + name=alias, + format=src_col.format, + array=np.array(cat_data[source]), + ) + ) vignet_col = fits.Column( name="VIGNET", @@ -342,10 +281,7 @@ def make_ldac_from_ascii( The input columns, ``NUMBER`` included, are copied unchanged; a ``VIGNET`` column (postage stamps extracted from the tile image) is - added to ``LDAC_OBJECTS``, and, when the catalogue has no windowed - positions, ``XWIN_IMAGE``, ``YWIN_IMAGE``, ``XWIN_WORLD``, - ``YWIN_WORLD`` and ``FLAGS_WIN`` measured on the tile - (:func:`measure_windowed_positions`). Given the catalogue's segmentation map, which + added to ``LDAC_OBJECTS``. Given the catalogue's segmentation map, which shares the tile's pixel grid, the map is relabelled to the catalogue's ``NUMBER`` (:func:`relabel_seg`) and used to set neighbours' pixels in each ``VIGNET`` to ``BIG``, as SExtractor does. @@ -409,23 +345,8 @@ def make_ldac_from_ascii( number=np.asarray(cat_data["NUMBER"]), ) - windowed = [] - if "XWIN_IMAGE" not in cat_data.colnames: - windowed = measure_windowed_positions( - cat_data, image_data, img_header, seg=seg - ) - if w_log and windowed: - flags = windowed[-1][1] - w_log.info( - "Measured windowed positions; FLAGS_WIN counts: " - + ", ".join( - f"{v}={c}" for v, c in zip(*np.unique(flags, - return_counts=True)) - ) - ) - ldac_imhead = _build_ldac_imhead(img_header) - ldac_objects = _build_ldac_objects(cat_data, vignets, windowed) + ldac_objects = _build_ldac_objects(cat_data, vignets) hdul_out = fits.HDUList([fits.PrimaryHDU(), ldac_imhead, ldac_objects]) hdul_out.writeto(output_cat_path, overwrite=True) diff --git a/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py b/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py deleted file mode 100644 index 4dbade857..000000000 --- a/src/shapepipe/modules/read_ext_sexcat_package/windowed_position.py +++ /dev/null @@ -1,442 +0,0 @@ -"""WINDOWED POSITION. - -SExtractor's windowed centroid (``XWIN_IMAGE``, ``YWIN_IMAGE``, -``FLAGS_WIN``), measured on a tile image at given starting positions, and the -background map SExtractor subtracts before measuring it. - -The centroid is a numpy port of ``compute_winpos`` in SExtractor 2.25.0 -(``src/winpos.c``), the version the tile SExtractor run uses; the -background is an approximation of its ``makeback`` (``src/back.c``). Both -work on 0-based pixel indices internally and take and return 1-based (FITS) -positions, as SExtractor does. - -:Author: Cail Daley - -""" - -import numpy as np -from scipy.interpolate import CubicSpline - -# SExtractor 2.25.0, src/winpos.h and src/growth.h. -WINPOS_NITERMAX = 16 # maximum number of steps -WINPOS_NSIG = 4 # measurement radius, in window sigmas -WINPOS_OVERSAMP = 11 # subpixels per pixel side on the aperture boundary -WINPOS_STEPMIN = 1e-4 # stop once the mean offset is below this, in pixels -WINPOS_FAC = 2.0 # step = WINPOS_FAC * mean offset (exact for a Gaussian) -GROWTH_MINHLRAD = 0.5 # minimum half-light radius, in pixels - -# FLAGS_WIN bits. 1, 2, 4 and 8 are SExtractor's own, each of which makes it -# revert to the isophotal barycentre; 16 and 32 are ShapePipe's. -FLAGS_WIN_SINGULAR = 1 # windowed second moments are singular -FLAGS_WIN_NEGATIVE_MOMENT = 2 # a windowed second moment is negative -FLAGS_WIN_NEGATIVE_FLUX = 4 # windowed flux is not positive -FLAGS_WIN_WANDERED = 8 # centroid left the isophotal 1-sigma ellipse -FLAGS_WIN_UNMEASURED = 16 # not measured: bad start, huge window, NaN sums -FLAGS_WIN_NOT_CONVERGED = 32 # still moving after 16 steps; position kept -FLAGS_WIN_FALLBACK = ( - FLAGS_WIN_SINGULAR - | FLAGS_WIN_NEGATIVE_MOMENT - | FLAGS_WIN_NEGATIVE_FLUX - | FLAGS_WIN_WANDERED - | FLAGS_WIN_UNMEASURED -) - -# Pixels per object and chunk, bounding the working arrays (~100 MB). -_CHUNK_PIXELS = 2_000_000 - -# Largest stamp half-width, in pixels: a window radius of 256 px, which a -# FLUX_RADIUS of about 75 px reaches. Larger windows are not measured -# (FLAGS_WIN 16), so a wild FLUX_RADIUS cannot allocate without bound. -MAX_HALF = 256 - - -def _natural_spline(nodes, at, axis): - """Natural cubic spline through equally spaced nodes 0, 1, ..., n - 1. - - Outside the nodes the end cubic continues, as in SExtractor's - ``subbackline``; a single node is constant. - """ - if nodes.shape[axis] == 1: - return np.repeat(nodes, len(at), axis=axis) - grid = np.arange(nodes.shape[axis]) - return CubicSpline(grid, nodes, axis=axis, bc_type="natural")(at) - - -def _mesh_mode(pix): - """SExtractor's background estimate for the pixels of one mesh. - - ``backstat`` clips at the mean +- 2 sigma, and ``backguess`` iterates a - 3 sigma clip about the median, then takes the mode estimate - 2.5 median - 1.5 mean unless the distribution is too skewed, when it - keeps the median. SExtractor does both on a quantised histogram; this - does them on the pixel values. - """ - mean, sigma = pix.mean(), pix.std() - pix = pix[np.abs(pix - mean) <= 2 * sigma] - lo, hi = -np.inf, np.inf - for _ in range(100): - sel = pix[(pix >= lo) & (pix <= hi)] - med, mean, sigma = np.median(sel), sel.mean(), sel.std() - new_lo, new_hi = med - 3 * sigma, med + 3 * sigma - if new_lo == lo and new_hi == hi: - break - lo, hi = new_lo, new_hi - if sigma > 0 and abs(mean - med) / sigma < 0.3: - return 2.5 * med - 1.5 * mean - return med - - -def sextractor_background(image, back_size=512, filter_size=9, good=None): - """Background map as SExtractor's ``BACK_TYPE AUTO`` estimates it. - - The image is cut into ``back_size`` meshes (the last row and column - smaller); each mesh with at least half its pixels good gets the clipped - mode of :func:`_mesh_mode`, a bad mesh the mean of its nearest good - meshes; a ``filter_size`` median filter, narrowed symmetrically at the - edges, smooths the mesh map (``BACK_FILTTHRESH`` 0); and a bicubic - natural spline through the mesh centres interpolates it to every pixel. - - Parameters - ---------- - image : numpy.ndarray - 2-D image, shape ``(ny, nx)`` - back_size : int, optional - ``BACK_SIZE``, the mesh side in pixels - filter_size : int, optional - ``BACK_FILTERSIZE``, the median-filter side in meshes - good : numpy.ndarray, optional - Boolean map of the pixels to use (SExtractor: weight above - threshold); non-finite pixels are never used - - Returns - ------- - numpy.ndarray - Background map, float32, shape ``(ny, nx)`` - - @sc [decision:detection.background_model] - """ - ny, nx = image.shape - nbx, nby = (nx - 1) // back_size + 1, (ny - 1) // back_size + 1 - good = np.isfinite(image) if good is None else good & np.isfinite(image) - back = np.full((nby, nbx), np.nan) - for j in range(nby): - for i in range(nbx): - cut = np.s_[j * back_size:(j + 1) * back_size, - i * back_size:(i + 1) * back_size] - pix = image[cut][good[cut]] - if pix.size >= 0.5 * image[cut].size: - back[j, i] = _mesh_mode(pix.astype(np.float64)) - - ok = np.argwhere(~np.isnan(back)) - for j, i in np.argwhere(np.isnan(back)): - if len(ok) == 0: - back[j, i] = 0.0 - continue - d2 = ((ok - (j, i)) ** 2).sum(1) - back[j, i] = back[tuple(ok[d2 == d2.min()].T)].mean() - - half = filter_size // 2 - filtered = np.empty_like(back) - for j in range(nby): - hy = min(half, j, nby - 1 - j) - for i in range(nbx): - hx = min(half, i, nbx - 1 - i) - filtered[j, i] = np.median( - back[j - hy:j + hy + 1, i - hx:i + hx + 1] - ) - - fy = np.arange(ny) / back_size - 0.5 - fx = np.arange(nx) / back_size - 0.5 - rows = _natural_spline(filtered, fy, axis=0) - out = np.empty((ny, nx), np.float32) - step = max(1, 4_000_000 // nx) - for y0 in range(0, ny, step): - out[y0:y0 + step] = _natural_spline(rows[y0:y0 + step], fx, axis=1) - return out - - -def _aperture_fraction(dx, dy, raper2): - """Fraction of each pixel's 11 x 11 subpixel grid inside the aperture. - - ``winpos.c`` counts the subpixel centres ``(dx + i/11, dy + j/11)``, - ``i, j`` in ``-5..5``, with squared radius below ``raper2``; this counts - them row by row in closed form. - """ - half = WINPOS_OVERSAMP // 2 - sub = np.arange(-half, half + 1) / WINPOS_OVERSAMP - rem = raper2[:, None] - (dy[:, None] + sub) ** 2 - s = np.sqrt(np.clip(rem, 0, None)) - lo = np.maximum( - np.floor(WINPOS_OVERSAMP * (-s - dx[:, None])) + 1, -half - ) - hi = np.minimum(np.ceil(WINPOS_OVERSAMP * (s - dx[:, None])) - 1, half) - count = np.where(rem > 0, np.clip(hi - lo + 1, 0, None), 0) - return count.sum(1) / WINPOS_OVERSAMP**2 - - -def _blanked(label, own): - """Whether pixels of ``label`` are blanked while measuring ``own``. - - SExtractor (``MASK_TYPE`` BLANK or CORRECT) sets every detected pixel - to -BIG as the scan passes it, and pastes an object's own pixels back - only while it measures that object: every footprint but the object's - own is blanked, including those no catalogue object claims. - """ - return (label != 0) & (label != own) - - -def _iterate(image, valid, mx, my, sig, seg=None, own=None, - n_iter_max=WINPOS_NITERMAX): - """Run ``compute_winpos``'s loop for objects sharing one stamp size. - - ``mx``, ``my`` are 0-based starting positions and are not modified. - Pixels not ``valid`` (SExtractor: weight below threshold) and, with - ``seg``, pixels :func:`_blanked` for the object take the value of - their mirror image through the current centre, or 0 where the mirror - is off the image, not valid or blanked: SExtractor's ``MASK_TYPE - CORRECT``. Returns the final 0-based positions, the windowed flux - ``tv``, the windowed second moments as SExtractor normalises them - after its last step, whether each object stopped on the step - criterion, and whether a sum went non-finite (the object is then - dropped at its last finite position). - """ - ny, nx = image.shape - n = len(mx) - mx, my = mx.astype(np.float64), my.astype(np.float64) - invtwosig2 = 1.0 / (2.0 * sig * sig) - raper = WINPOS_NSIG * sig - raper2 = raper * raper - rintlim2 = np.clip(raper - 0.75, 0, None) ** 2 - rextlim2 = (raper + 0.75) ** 2 - half = min(int(np.ceil(raper.max())) + 2, max(nx, ny)) - offs = np.arange(-half, half + 1) - - tv = np.zeros(n) - mom = np.zeros((3, n)) - converged = np.zeros(n, bool) - failed = np.zeros(n, bool) - active = np.arange(n) - for _ in range(n_iter_max): - if active.size == 0: - break - amx, amy = mx[active], my[active] - xs = np.rint(amx).astype(np.int64)[:, None] + offs - ys = np.rint(amy).astype(np.int64)[:, None] + offs - # The C box: [(int)(m - raper + 0.499999), (int)(m + raper - # + 1.499999)) clipped to the image; (int) truncates, and the - # clip makes truncation and floor agree. - r = raper[active][:, None] - in_x = (xs >= np.clip(np.trunc(amx[:, None] - r + 0.499999), 0, None)) - in_x &= xs < np.clip(np.trunc(amx[:, None] + r + 1.499999), None, nx) - in_y = (ys >= np.clip(np.trunc(amy[:, None] - r + 0.499999), 0, None)) - in_y &= ys < np.clip(np.trunc(amy[:, None] + r + 1.499999), None, ny) - - dx = (xs - amx[:, None])[:, None, :] - dy = (ys - amy[:, None])[:, :, None] - r2 = dx * dx + dy * dy - area = (r2 < rextlim2[active, None, None]).astype(np.float64) - ring = (r2 > rintlim2[active, None, None]) & (area > 0) - if ring.any(): - k, iy, ix = np.nonzero(ring) - area[k, iy, ix] = _aperture_fraction( - dx[k, 0, ix], dy[k, iy, 0], raper2[active][k] - ) - area *= in_y[:, :, None] & in_x[:, None, :] - cy = np.clip(ys, 0, ny - 1)[:, :, None] - cx = np.clip(xs, 0, nx - 1)[:, None, :] - pix = image[cy, cx].astype(np.float64) - mask = ~valid[cy, cx] - if seg is not None: - mask |= _blanked(seg[cy, cx], own[active, None, None]) - mask &= area > 0 - if mask.any(): - k, iy, ix = np.nonzero(mask) - # (int)(2 m + 0.49999 - x), truncating as C does. - x2 = np.trunc(2 * amx[k] + 0.49999 - xs[k, ix]).astype(np.int64) - y2 = np.trunc(2 * amy[k] + 0.49999 - ys[k, iy]).astype(np.int64) - inside = (x2 >= 0) & (x2 < nx) & (y2 >= 0) & (y2 < ny) - x2, y2 = np.clip(x2, 0, nx - 1), np.clip(y2, 0, ny - 1) - usable = inside & valid[y2, x2] - if seg is not None: - usable &= ~_blanked(seg[y2, x2], own[active][k]) - pix[k, iy, ix] = np.where(usable, image[y2, x2], 0.0) - # Samples outside the aperture are 0, whatever the image holds. - pix = np.where(area > 0, pix, 0.0) - locpix = area * np.exp(-r2 * invtwosig2[active, None, None]) * pix - - atv = locpix.sum((1, 2)) - sx = (locpix * dx).sum((1, 2)) - sy = (locpix * dy).sum((1, 2)) - sxx = (locpix * dx * dx).sum((1, 2)) - syy = (locpix * dy * dy).sum((1, 2)) - sxy = (locpix * dx * dy).sum((1, 2)) - finite = np.isfinite(atv) & np.isfinite(sx) & np.isfinite(sy) - failed[active[~finite]] = True - pos = finite & (atv > 0) - # SExtractor divides the offsets by tv only when tv > 0, and - # normalises the moments by tv after the loop either way. - with np.errstate(divide="ignore", invalid="ignore"): - dxpos = np.where(pos, sx / np.where(pos, atv, 1.0), sx) - dypos = np.where(pos, sy / np.where(pos, atv, 1.0), sy) - mom[0, active] = sxx / atv - dxpos**2 - mom[1, active] = syy / atv - dypos**2 - mom[2, active] = sxy / atv - dxpos * dypos - tv[active] = atv - mx[active] += np.where(pos, dxpos * WINPOS_FAC, 0.0) - my[active] += np.where(pos, dypos * WINPOS_FAC, 0.0) - done = pos & (dxpos**2 + dypos**2 < WINPOS_STEPMIN**2) - converged[active[done]] = True - active = active[pos & ~done] - return mx, my, tv, mom, converged, failed - - -def windowed_positions(image, x_image, y_image, flux_radius, seg=None, - number=None, cxx=None, cyy=None, cxy=None, - valid=None): - """SExtractor's windowed centroid, started from given positions. - - For each object, a Gaussian window of sigma - ``2 * hl / 2.35`` (``hl`` = ``flux_radius``, at least 0.5 pixel) - weights the image within ``WINPOS_NSIG`` sigmas, the aperture edge - subsampled 11 x 11; the centre moves by ``WINPOS_FAC`` times the - weighted mean offset, up to ``WINPOS_NITERMAX`` times, until that - offset is below ``WINPOS_STEPMIN``. Invalid pixels and, given the - segmentation map, neighbours' footprints take the mirror image of the - object's side, as SExtractor's ``MASK_TYPE CORRECT`` does. Where - SExtractor would revert to the isophotal position (``FLAGS_WIN`` 1, 2, - 4, 8) this returns the input position, and so it does where there is - nothing to measure, the window is wider than ``MAX_HALF`` or a sum is - not finite (16). A ``flux_radius`` that is not positive or not finite - gives an empty aperture, so zero windowed flux (4). The tile weight map - is not used: SExtractor's ``WEIGHT_TYPE MAP_WEIGHT`` thresholds and - ``INTERP_TYPE ALL`` interpolation of zero-weight pixels are not - emulated; pixels outside ``valid`` are mirrored instead. - A centroid still moving after the last step keeps its position, as - SExtractor's does, and is flagged 32. - - Parameters - ---------- - image : numpy.ndarray - Background-subtracted image, shape ``(ny, nx)`` - x_image, y_image : array_like - 1-based starting positions: SExtractor's ``X_IMAGE``, ``Y_IMAGE`` - flux_radius : array_like - Half-light radius, SExtractor's ``FLUX_RADIUS`` for - ``PHOT_FLUXFRAC`` 0.5, in pixels - seg : numpy.ndarray, optional - Segmentation map on the image grid, 0 for sky, labelled with - ``number`` (:func:`read_ext_sexcat.relabel_seg`) - number : array_like, optional - Each object's label in ``seg``, required with it - cxx, cyy, cxy : array_like, optional - Isophotal ellipse (``CXX_IMAGE`` ...) for the flag-8 test; without - them bit 8 is never set - valid : numpy.ndarray, optional - Boolean map of the pixels holding data (SExtractor: weight above - threshold); non-finite pixels are never valid - - Returns - ------- - numpy.ndarray - ``XWIN_IMAGE``, 1-based - numpy.ndarray - ``YWIN_IMAGE``, 1-based - numpy.ndarray - ``FLAGS_WIN``, int16 - - @sc [decision:preparation.object_position_columns] - """ - ny, nx = image.shape - x0 = np.asarray(x_image, dtype=np.float64) - y0 = np.asarray(y_image, dtype=np.float64) - fr = np.asarray(flux_radius, dtype=np.float64) - n = len(x0) - flags = np.zeros(n, np.int16) - xwin, ywin = x0.copy(), y0.copy() - - ok = np.isfinite(x0) & np.isfinite(y0) - ok &= (x0 >= 0.5) & (x0 < nx + 0.5) & (y0 >= 0.5) & (y0 < ny + 0.5) - empty = ok & ~(fr > 0) - flags[empty] |= FLAGS_WIN_NEGATIVE_FLUX - sig = np.maximum(np.where(fr > 0, fr, 1.0), GROWTH_MINHLRAD) * 2 / 2.35 - half = np.ceil(WINPOS_NSIG * sig).astype(np.int64) + 2 - ok &= ~empty & (half <= MAX_HALF) - flags[~ok & ~empty] |= FLAGS_WIN_UNMEASURED - # A box wider than the image holds nothing more than the image. - half = np.minimum(half, max(nx, ny)) - - finite = np.isfinite(image) - valid = finite if valid is None else valid & finite - own = None if seg is None else np.asarray(number) - tv = np.zeros(n) - mom = np.zeros((3, n)) - failed = np.zeros(n, bool) - for h in np.unique(half[ok]): - members = np.flatnonzero(ok & (half == h)) - size = (2 * h + 1) ** 2 - for c0 in range(0, len(members), max(1, _CHUNK_PIXELS // size)): - idx = members[c0:c0 + max(1, _CHUNK_PIXELS // size)] - mx, my, tv[idx], mom[:, idx], conv, failed[idx] = _iterate( - image, valid, x0[idx] - 1.0, y0[idx] - 1.0, sig[idx], seg, - None if own is None else own[idx], - ) - xwin[idx], ywin[idx] = mx + 1.0, my + 1.0 - stopped = conv | (tv[idx] <= 0) | failed[idx] - flags[idx[~stopped]] |= FLAGS_WIN_NOT_CONVERGED - - flags[failed] |= FLAGS_WIN_UNMEASURED - mx2, my2, mxy = mom - measured = ok & ~failed - flags[measured & (mx2 * my2 - mxy * mxy < 0)] |= FLAGS_WIN_SINGULAR - flags[measured & ((mx2 < 0) | (my2 < 0))] |= FLAGS_WIN_NEGATIVE_MOMENT - flags[measured & (tv <= 0)] |= FLAGS_WIN_NEGATIVE_FLUX - if cxx is not None: - dx, dy = xwin - x0, ywin - y0 - outside = ( - np.asarray(cxx) * dx * dx - + np.asarray(cyy) * dy * dy - + np.asarray(cxy) * dx * dy - ) > 1.0 - flags[measured & (dx * dx > 1) & (dy * dy > 1) & outside] |= ( - FLAGS_WIN_WANDERED - ) - revert = (flags & FLAGS_WIN_FALLBACK) != 0 - xwin[revert], ywin[revert] = x0[revert], y0[revert] - return xwin, ywin, flags - - -def isophotal_ellipse(a_world, b_world, theta_j2000, wcs, x_image, y_image): - """``CXX_IMAGE``, ``CYY_IMAGE``, ``CXY_IMAGE`` from the world ellipse. - - Maps the isophotal ellipse (``A_WORLD``, ``B_WORLD`` in degrees, - ``THETA_J2000`` east of north) to pixels through the local Jacobian of - ``wcs`` at each 1-based position, and returns its inverse-covariance - coefficients in SExtractor's convention - (``CXX x^2 + CYY y^2 + CXY x y = 1`` on the 1-sigma ellipse). - """ - x = np.asarray(x_image, dtype=np.float64) - y = np.asarray(y_image, dtype=np.float64) - eps = 0.5 - ra, dec = wcs.all_pix2world( - np.concatenate([x, x + eps, x]), np.concatenate([y, y, y + eps]), 1 - ) - n = len(x) - cosd = np.cos(np.deg2rad(dec[:n])) - # d(east, north)/d(x, y), degrees per pixel; east = +RA. - dra = (ra[n:] - np.tile(ra[:n], 2) + 180.0) % 360.0 - 180.0 - ddec = dec[n:] - np.tile(dec[:n], 2) - jac = np.empty((n, 2, 2)) - jac[:, 0, 0], jac[:, 0, 1] = dra[:n] * cosd / eps, dra[n:] * cosd / eps - jac[:, 1, 0], jac[:, 1, 1] = ddec[:n] / eps, ddec[n:] / eps - - theta = np.deg2rad(np.asarray(theta_j2000, dtype=np.float64)) - u = np.stack([np.sin(theta), np.cos(theta)], -1) - v = np.stack([np.cos(theta), -np.sin(theta)], -1) - a2 = np.asarray(a_world, dtype=np.float64) ** 2 - b2 = np.asarray(b_world, dtype=np.float64) ** 2 - cov_w = (a2[:, None, None] * u[:, :, None] * u[:, None, :] - + b2[:, None, None] * v[:, :, None] * v[:, None, :]) - jinv = np.linalg.inv(jac) - cov = jinv @ cov_w @ np.swapaxes(jinv, 1, 2) - icov = np.linalg.inv(cov) - return icov[:, 0, 0], icov[:, 1, 1], 2 * icov[:, 0, 1] diff --git a/src/shapepipe/modules/read_ext_sexcat_runner.py b/src/shapepipe/modules/read_ext_sexcat_runner.py index 60249fd73..17e23163d 100644 --- a/src/shapepipe/modules/read_ext_sexcat_runner.py +++ b/src/shapepipe/modules/read_ext_sexcat_runner.py @@ -32,11 +32,8 @@ def read_ext_sexcat_runner( a FITS-LDAC catalogue compatible with downstream ShapePipe modules. The inputs are the catalogue and the tile image, then the catalogue's segmentation map if SEGMENTATION = True, then the WCS log if - MAKE_POST_PROCESS = True. The windowed positions (XWIN_*, FLAGS_WIN) - are measured on the tile image at each catalogue object. With the - segmentation map, neighbours' pixels in VIGNET are set to -1e30, and - the windowed centroid masks neighbours as SExtractor does; the map - itself is not written out; the + MAKE_POST_PROCESS = True. With the segmentation map, neighbours' pixels + in VIGNET are set to -1e30; the map itself is not written out; the only output is the FITS-LDAC ``sexcat.fits``. MAKE_POST_PROCESS runs the multi-epoch post-processing that adds per-exposure HDUs. """ diff --git a/tests/module/data/xwin_sextractor_186307.npz b/tests/module/data/xwin_sextractor_186307.npz deleted file mode 100644 index 7f5242d60ba41e405353d8fb2dc6fdbffae84362..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23951 zcmc$`bC4#Bmbcqw+qP|V**3at+qP}nRb962>auOy#@#dL^vpSXcFf%G&Np{5q9XsP zhF>_4W4y{u}Z)eS0$#Q%4;$D?Jk$0Er&!-i<$BnMC4p^jcIUf%EByxkD3Tx^=x z*hCKlrD`q->~cMPtJ1`l*8AmDWed-C@NB7jENZHo-q#<0MczIvegNnAZ zaYFSK&87Q^(Bi-bJz3CaO6NMIdPsdoRJbU^Hgk(}IIjVM0lusy)9`R7KOPUNPZ`SJ z>~jzhm?GjWB?l;LolH$XCTiA5qEuK1;qrb-Ml69r|2035>+jV};H^yglBVMOZIr=u zbxSn)B~!5N212fkO|oz$tD?s(J?lqLz-ZkbR2$^JoH?_$aMgOm%g%qL2` zb5w$AB$6cySJ1sItm4-(4p_!Bf_d3Ff@_8?YC6f{^(-ZjnNAAOs|@Iw=KxG_IPJ4rln>9jPPBMtbVtr z@|N!8*VDqLr(KZTUKTsdiG4^(A@D!bd|^dxLh#pHDeg-^OV?+5Z>=ODmfSHe7JRI# zHm%`KaS$@OTg70?MG73jKn$I&!xNi1C7H;q#FR&Urtj!Q6X(()%Bv-N30+8d+`0ju zzpiU)L|{$mBERw5u~|l_!TKTTZ1C6sG|PGrysFTqc`RgvnrP{D9(clQ$>T%PCHv|P zsLOFWW1(!Wy;4~ z7hNCewktp^QA*$&-MC2@$srNKjwKWZc8fbUv6gg*hV0=f`)+wu>WMty%-$m=B!bn+ zHU9v%A-(FKj53&B&4NpNICb@NfbK$F2?N86GC+xdTYl&Own>&8M(qZ_?nDy6J}e0| znVsletp(mMLFOZu`b7;@p#X zs;2qNClc(KjeW(A<6^5JNyft|u}heqn`mdo#DaFa#-x~CO%Xxk#Y14}mw~OK2M)H+ zxi7HC0DP?*CefA8niqS&tem>E3qqIJ~M~2rds>!M8VJ;8l?RzBRu}rBCRo6&H3T z?56?_Xw~Ss6~)#g4E0PZNg9B`w-w^3owFZ$=Vn~&Y|*CQxFk*8b|i{@UeOgBsRJP} zS&g)t*=FxZD^5;sohwvrL32#a{H77r-ru&H@#Vtx-Eh4--h6+)9H5a?wgnM<;ufTg)EX z-suJycGT%fp+=k0uvaikksvbOi0~NCxa*kc?owLE)|8t>7gZd`tn}n`wMXD?V@F*I zd=hN(Bs9&k3|^^41z$CSLv+FrqqNuO*3!o}Z%LBisK;Vq1437f-fnOK=aO3520+0` z|Bff<@}4)9nAtigWrJaenF)r@uZ3D)q!==z98*|y_R^rJ3jQhRgJ8TssY}d;fDCn% z>j2O>I}9UR$}1UMs8HD+do?C_FdBY-+@U|ul^qgVMqO3E;$O{m(Q9+|VHS0|V;SC37$!7Ccwlt(?-$A=mRy-uY->Tg%+dXY*R;wPbG{-r zjM-6k-cG@HIX%D9ztXtc+D2+fInn zZZ#pBy`J*f{DcS$9CM)Bv&gk;gMrD=^sJI|mk3dG}`q#kH3-k(3`|q8e#z`F76-vg%X` zEe)Q;Zf41)P{j${dBab9b~c=<+8T$+i;?(wj5f`!4`MX&*0URiD%NSAcOn2Vn~^T5 z9YwRk-02?aZXqYw(S|W^?U3dnz26GUtz1Rkhie3Kx4rEXFXkeT+8o=BJQn?fAmJMU zUkA^s`XdB~&xtVXR(Ij~A_@VD5;R`73*Q}*Pq2FS_XfB!yFL^o@|Ya3-2x2ao-Hsp zkqy%F`8{@sEtm~+dtz%HnY{7Vq0B57P$q^U>!4QnaPmQ_=E5PPU|-aDDEv zipPMnd6NAWd?-vfa{o(D1`hcI*F)3&e!d#lS%b@V51d>|5c=Y9!V_!rV+`xPgW+$2@m1`+4@v=dQ7QE#lG#7Ozu zbBgGcFK*8=F6oCBLTl}7lv{>dQu}4wl+G$-CC56OPv>ctp zct}JS$#Du9kZ?2;V$?#w7fg|2PDqjBbWX@dNJBu{U;uzfU9q3;N@8uWp6)f}!$n}t z=9Zk-%K}4%lGg=Vv14I zOtIg-{c}ltHIXpCOZta-)xlBE-tl))sedm<^*=AlIPbqIYKZe+3qt-z3yN?0TygL% zA1B+?{}@d5l8NKbV4bc|Zok2h{=L^muC_)7jz)&R+5RZ#Uk~N~E}O`|v*nnasry)P z(4R-NFBg{`{F$sQtoHIZ8R9>XSz4OeI{e#^ANhA6x&Hzz|NnKYb}cliked7 zkM(}lTL3v9puZfUEk9^rq@1ANpDy86x`|xhHb1(&zwTha`@=qeoWY&1*G^q+S((-j zC=Kg}#g)mVPJV@*7cr%KBIt-ZKo#`9yzmk;^FbyPYXezqX$du1gNi7MQ>w@bHEA&o zYQ_h2cPj3XLHY3Oq+qLTT!vGA&w5$WS~IQLC6)6bm#$RhM&vF@v*lp*YY4*b4?Y2k z#laQ4uP@#fTm!5qtuWBLd?_q-%i7+n=%*)Z!WAn)}vBVo7uj+Y)@l9!dh)_`k4kS(iZ$5)N*zgfc2Dy-1LFA%5E zzBQoJUtPoDqG-D&%_k0pWcl^+ji4U+q>u>ly-k9RYaGt^xYX%>$iuCz^$RlF>LG9y6zcQ#CzmmH!=X$xHBB3E^?jX1c1qoHOEkYM$f#actD_!`+7N z7C-pFkQNUTdY+RqTg&s*jy1XG#pdr>)>bmFYK{s(B$s zX1xjn%=%S0U4W2R;Q~740Q~qeLA_y$J3`Qc$lvuF)pTjb%t6kM4(aM5%^(H3yvrU{ zCDR=6;KkT059E4}VH9YaGwZm$2C}ZL*2Ox8yzJDYu!)Wq3JT9Dg>Z&XqN?>wwX54r zVDhoL8lk~K44e2gK5tenxU^=HZ=X$j$BdJX{vy(+M1gc=D@W<1Ot(xh(%D(tlpmLo z?YG@9A-odsO0O#kz^0`EY@BqcXko^5K4Q~1m?0{?O)9|2lD$c}nAAkq17$0hxOT6P zby5v4ut>_-mdc}{deg&%VW>sTgryA{e0VfnbwiXM{y9B063M*Icr+VxF1IC$1-S?7 zJ6hzzPOKLxt4glGuT9!ZX!d&X?&)soirOhp+IzK#YT(r8E$5!(Z?hQ~H-G?Y=ta|& zj9BvG%;+gJ9G^Bh7h;~pki23*F4yYn$6bDE%zeqz;>Lx@_{u>#aIo_rPlg;k*~6F9 zxDO`_Lg-o)7U0~IX_GzJP;azj2p598IqK}aV_#ZL7Lk$Ij6#3YGZn9}R> zde@qpSlFQ^qdfwY#VQx;!^iKokT2O7l^<^TRPI;q4>64w2%e4UeAELJc%F2^c%LT( z_1A&YwK#);(jvIcl-7Ky1EXSxYsyb2@p#N%XM9a-Y%2N2LehO`yJ2o%ZdDRN+xIeF zZHlf~h$c*kTqK#NlaYSkNg@C~lIR&f*~`!p@+FzQbUo$5tY^wY)~gI7o73TnmS%5! zo4Jy%8H*;OY#6!>Ij1(g)$yBczKc1H&FTla#~(bQGB0Eoe4Jg4$YT3})E_3~Y*_}B zN-j(%8cR;Jh4Am4kb@H~Dq7?*L}u?c5g;h!Z?3=9SPsEw(kg*zdIL1f?YB6N17T;r zf_l$;e480Gcg$`A%)zMx8NT2GE>?`*d*wlhNZ%4lbYBeB|GfJKRH@KYH2ZDc3#L>s z2@AzbJg+&#Ag`wtIa6tgzq*JhSo~+e@LYobcnyWpqo`4v!zA?Qu5!NVbT{(Vdsbgt z4yV3a7NyRkSXNnAwg^z-t-Ogegr;{zNjROan4KryfR{7z;j4bw}yu1iriK z5GKvUuUyhvY|@jH{E6_Od1VQNEfDnSK9^O`B2CUWY)=&BuIPYi!Gtzyr9&Mr^7g02 zc(@p21}hhuE(Gq@w>nnVpY2IJ-unI|YwHZl4WPVi&AvOYfb5&8aE$RRu_xt%)gB|$ zYPYARhB<<)S*5;wDXh=-M-1Lj(F0A3SL1{GO!C^Xfr<|%(>RY$;=ekPGw=4%E6laW zJ{r?|f6!n~NowmJdf1EgspuFN-$-A(uUWZrv?4>Q2TxbBC7yTZgQ$qw@Xb4$tv>Cd zH?Wy0?$w#zmNENfscnufE{WDn`#J_A2BRBUN>zf9&5=#sv=HU{62E2XI~WqFNm+94 zvoPn)^Z;_U8wRp>%#&uiERTp+_z&TJXV>IR(tkqNW>t@AZ?YzgbT{BR!fxK6C7+CEj@4*$%|j z6}_AH8DKbSkH%?r361~Lsk=gGl_WUN8TxTChu;i<~lBB$#5t{W6ZR9 zHh8fGZ>0_^lT561aMxunp#jx@!G?S-3q-V0%DGnJaT$-z;eQy5DL(_C+*|0CJczFm zHD?GXg5-aS;e2V0~zbMKG|=^N`K>iKR+f|$AKE}%2wvo z>5tlOoY0gu_S(I}jCp(|i2+x9T7bV@RWh&Kkq*BW3$?wD_)oeQq?b$*4bt=E@Zprd+0+}m!S z?CPW7rA40T9nj`M6PPNRYNh_O(i&#!iJTWa*`Zc~-TZ^u_`R|u6cscdEEv|x<-^Nfb&;RkN<#K*g?(K|_Db$fpm)~oL$9a{3E6CA+uZa) zp)j8Yf`X8vQ-aA2^77Fdo-c&4bvTBr$)YXo@pSY2z?B{Kkohj&z$-7>w(@7FubvS@ zRF&1a)HDHVLA*m820Hk6vU<{f=B$A}E&v)=_VOe1f~E5_{z%r9wnc{|Tx-oK;u5QU z@@H(WjT;dDR_AI1ZX4+!2i>~i@(^Ms)&r?FQv_zqsabDzL5J^e{1Tiw^2^&n_E{ys zPgm+a*JXNK6)&;u&S|zZqm0vO7s5gtc5L^V^SaXI^ckl)5T5_{&!FDXl{>H;krZ&gEm~eo4UxX<4%ys-I zZ)XDydz5C$;6}F#r;djqAT4ct7+b>@xASFTxmws_C&8p0t7k$M5r+bB+5KtG8U#iK zpzW2~K*1z)sYSvpwf75BHnz->3b*v`3Zyobdn59_6Oz?7q!F`>o=)>Dkj|0R2fq%9`_ye`EcNqWw)m z{^#AXDtOO24EMDS7TQFcnx1^g`Op0-K!W)>Nfxs3f51xZc>`%^Mh0NMh6YCunVPYWEtNZ*&KiQR~1j8bhy90WetduP*`(x(Gmp@N}5wMqa{ zge9V!_Agd;5p`&E0Z(Lon}r)rctN#+zo0Il!u#M@=pODN`9j?~-3Pmza|(p5^{mNY zD)(;0rgC}nzw2kdjzlL6c#PO~X~I_GCQ4Iiq`B)h3S`9^7i(-wyi=kDZ44q4lQu_d z@c9J7?i4Ar@Zm(GJ0~+<1!AMhmTS5ZSmwTs&jHVnTsT0wSIHeJ*~Q4~>u+=<)`#G2 z?e+5z;no~2pE5jXN)jnr$KR2`7=>WQd?ni`tEq;cH9DwS;|fs0_^x=guGdBC%JAu^ zoUF>RAX--a5o8`wh)EPb)OzhpT+FCHDKKwx9Ma4wB{$c5K~;T(a}cwAa+Rs~a8mOA zRM|2QnC0?e>_${9+Z5-VUKX{@ZJJl^K}*gajgJPDG%Gd}ZB>4~ zT?So@2>50D96)Xn0_OYxTfX;f`2bAH4$e?X5RK8ZVyqW-oe~mWiRl6ZWx1iJI}nvJ zUHG|gU!%<1G#9G&-?c$z#$c9Cj2=L-XHy8okx*t8luh(RA7<<}=xV47edq0)?|xz1 zBowKBd+FvC<t z1LfwN^X1qQMa;F>e=B)bW-1hp;v&91vo7;#CrPT~w$H%cQE#Glx1ie_u66vv4nGUy z>s6;F`NVy+lQjb7XlKO(!6JPv;m9y}vcSyaU5_ed?tIrJk^sY0Hc>Nt&a|Jp=?59o z`Ti_NIGABWvd9oJj#lJwbk<&a)I^Q6wleCVL6g1 zoi;k;GQrwcD<1~cSf+1FBc|66bJWKh&u<461+%RpQXMva+E5knDu>-J>-NkohLOor zUlSY>vm~`-Iy`kG8U}}L?PRqB@G?4*aS_LC$7VnK(EOo5jAjA8t{l%V=WWSTwg#Ms zx&`Fci_w+P2lW=Sv38yh2@t4(W)GY*o7}G^?8;S*6!KB9K)J1F#p`VHf(RGl} zw?d6|*;;V`I!t-l&5B|a<^&Qc-r!_%hZ^%3val|;emhYwY>=*)!YYV$Qe0->84yI% zEY*)Fy3KWs8!H}$Fwap25BRx6PtgKb4BZ|4gY3{CWP<#2u?y+#A&WwaDqJWY`0(}h zr)lkQ0Rt0T=q82!2F6RWvuM`U6M(^#U zy)J^Q_Y$QYM8U4|J=YJHmt0Sm#A?JdqIL`q#80D8D~|dazZL!PFes7l5`@w0c6liS z@^t%vQ6*vhqF4IMa~~+D?4i5s8F!^x%>dZYiM1x20Dajx0B3_!7XETtgHD%jUlPM@{@-Kse5tkOb2;i=pD50SCwCN1pQ1`cw8wpbY8qZ9j94{xe zUtuYl5zqW8)u+8fgfaflha^8*m%xK@If&ReNAcV=VAs~L38)XFufx3nN|(s)j@Kj4ha$og-r<$v5BVDlDf3|De$0V-FMlx+}1(}vUN zW`J?#nm)1&6vNrK^O$dMNQQ^tRm+j`J+_^;`;U zp4_6jbLs}}a?AWMN^-#eH-a73lGVI7*S>|Ix4p2Z~X$Fbt*FF^_E8^SxR%k?>mJ5?4&EyRT) zlockgdtymE;NBnadW~rCclvSt6Sq{J$y8|$w}>YK;)jWCfu3v|gWcj4JBqAu2U|O~ z11@H6M?CfZ6mAndMpXd6u!j9>dnzKQH5+DE=I-HPU|d=p$JuH*zkRvHNm~St_XTlu z0PRNKosae&C+>sd)U=!ZbaGG?KSG{>FuYNdie24m8u5oFpZo_zgHV0Wys<3*+`}Mo zNkL#`0Y5xjN6s5nDIz-shEj@(kZ1_j{T(6H&!3aIbyW!9iI+v;C1}U1$t<41hRjcg zak!fch=UcI%-&TWY6>qjzA=zVJ@yge-hG}}>{()YmVUJRuuAgJrO3kP>r>4+alT$O z-7L{*?~uG*KD3@{TT)5QiYL#5N%$7nvTg3!9m}N*2R!YQ6A3PQ3|UZWNoOE!RMbVU zPo#$R`}BcJ0H5a1C`FO#b7B{am>BP9S=TIEnVT(J3D61er-YC$kZb&(V`}13(()8C4{DML>!WV%sig zbVcl>eH)IB`pqIjYJwBS==QGASRxJtz`SvLWpLlxT^Oe(==9Q@m^wWS`$9giF=W_F zcUBOb#%YgtaVu;Hm(qT)duT(Ov~}&=C)$^7Uj=#2G{8!$9nY5RB8!DL>y=?5t%pSb zdV5P^w`y01^h2&F8In!ecm6CDwF_M(QGPa_qn*`y#Za$)jB9ij-P?`{*38?4udE)~ z1Fg)tnMGQ2ohvCY+$7hnd+m%xA^W8TmtfGJcI#^331&f`WMqW3MzB7PkPav;cBaON ztKbDTyYO_<9~6;TOO`dDb+@hbK=`Qh&j#6`!s_>Xs7C**q53am((XSOkNza>i+ADWojIkGdTkh&0?Wt}29?FB&(rOP(J7&$5JeVkGw){V#psFFo_W zy$Sq>EhYSN{@%BIOc}F)V_+fzRdE zGco-SdsHdGF{0m$*ni3RKP{iMGg}T9jn02OT==uP4ew7a{73o3{VTlx1rLQVNg4g! zGOWK~V*FTM$^dO{rz-x?qusmw0S{J5h2!%htV?tAd=P%>EV%^ef4 z^z2~S;Mj6O-ugDvm7gt}Ht4;qdEe*}BRrX-L^`r_9Rym;jLREt4^4(%341cA3Qk@+ zqwuDFIp|Wtp59TymhQX$tt zx3<3GUzf%cD9O|vR4FG=HlntX;~~`KdWt{!L20o6$~roau3GarTJp)vmTKl1>WqiD zH^(76a=HtD-*C_~v^(0DPo>Y(?gq+>an>(2MO?Fwa2bnGDMMG)RL7~qAk5vMJ0zLR zgfj#3qo8=Ab5AJOQz3+Mt;lsQkab7tiTzv^JYak?rRLF@XU$DH#l4kq>sc92csM0* z3KpSx?u3~CQ-{A(4j3t%^M`N2cO7)jv^91VUr8M@vq_qK;b!FPCwrGoP6q%w*tvky zQ=#_V6>Hb85NH+IXIic`70?u|Y$Z>ct4Uw*QLV9mTk>oWgkw^xJ%140=SBrNXWs$K}-V^{~Y zuAOSVqa=uqk(uOHpQ(MeBKo1sa@CpDVd?xk@;JhxxPAWb-5){6!ovzptoL?l}D`LzOTK$u6KlF&1YXZ1r ziPC=NA%Pj+BjBY`nJFi0wopxbQt~0;q&d2rzGW!m@~B~Ei%DH=?8U@+!y;K>0m%ZA zmn-~j%&|}L=q6V88Zf~%0=D#iod138r(Dg z<~Jh|V}Zmc{XDu`mpa83fRx9P-Qx^(E`!hS?v1&8BiuCz*hjCr6Sl^ zx&UKCM%y>}9&&d^%xH*8&qc|6FIfPP#Zv52CwupuIQO^wnp!$w+Oh z(6!_~_Jag#obgNVNm82P&~-#2<_-{Lqm(oW4%#6cbV}K{2F^%1&S0S}T=b1iP(Ip` z1etyxMDiF2B3I4w_q!XIW~@QJd$OP9OrP)|9O^sm<<)CihEcu)CEm!RtWhb$+x_r2 zUV2=RE4pXl)awfWGTAQ~>Hx*Me5(ZT6tD^KmWQ5xVZ07i%->ny?Atb>alCvQI$?X= z7UyFcYu@S`jsZ5@RlK$oij(`$18O`nW;AHxH!X`l0{V#gQQ3{T*h=O{jcv^RbTvDg zr7GUC1j@GX6mUYLn7zLUGgnxy=kb9~9bN@I;l(1?k$Lkb+cxTXrarm@P2+n13XMwZ z4$T%*DLPm&>e|+W9$_ef&wmR793f@8>J1sr5)#-9iOEHp2frSz6!(0AIp3li))|5n zG}XpyRzz(;F#1|UzzC@MgiX+@m$95~@j>tS#(+F>2K9^SpjCTF$y(kAJ1RYSkNIxt z+O8RZwrY_di^HV?K0QrjCHvv@#%ZyK<}Ll1p( z#zE)=3&ekJ*-?U5cdV{^Js3|%VkbJYS~nwWJX7+)c4Qyo&fcyl|1M&ux5y{A!G|qJ zO@oHHdl3L@Z5aB|WX=wWh}O%b93S$SLn9||9413pKt>JCQK*cFJaWJ*0V!{%$MhVX zV4Kex2lU|`zF1;dUacE_9J(og&V??~5e-Vl?dHdsDFE1&d4jCiH@dF{WEVHM(9NF{ zHYAjH&l|~zWrj@G12U*Byl^U~Ir$XbIEDLjUG$&DIM5Y##?RU)NCubs6?zQDX&HHX zi=Oi>*`STQObnCTl&LX1)6*};sJNTN#>tmn^kaJ5$zPfQj~Y0E4LS=C+00Q}ZAjo5 z#`N#cmEq7dhc~Fr3_sjSV$8VRAU2U6=e(3ryag+hP&b#u5b9<5y7b8A7{&c*6k1yr z76Ir^s|I8%L2527(QH@BOp6Lc=+@f_lCpv?Kega(xU*DPqw_AOkaRDM(89rC2ojvH zh9tu+7`)%5eM?>Db!IoGv1zFy#a{uak7za5Y9R?73~%hxfn|%8#Y{aI%w7l5H`q@NiT$nJg#-O%9!c=fe(l~wxLK2QAkHgF1lrbrX=U`jsc683Nf;vR=V7|`l;KF)8Oez| z5^Hb@EB8?@JmNYhP{9CTpXs4Dv+LC2Gwg&uf(NyhK1*MOy_7N{4*+ zUGf*b+p@+g%NBplyI|z0`wjcO6;?+THGl?;?EJ4czVaXROEjMlj|;W7#)%Z>+~vN{ zuo|YeU;L;K0seC=KI(3~K`|-~{NJ|>QuD?}Z41acDBmSRy*vVyCC+w2n7$AI-V%xXA_=f9PEg@^ z*>&64DNHA(uRoolFH!-^j)m)1FYU?D3c2fc<|<~37Dk?Hi%`x|ExB*A5JKP$bvS9nEd(P=K|W=nNQX+&AB%MC(=1xHrX@nuLa&b4ICwt23Y#Bq)~I-{M3 z;t?lu#pb0GpQ`U&{;m`FvyGHIdw~AH4HwzCNH?+wuWD8vWXaPjb-KovZ2f*%&=`(D zZ{D7jW*Ycw*#k6}6~TV^Nc>bmR?haEcJZ_kz}Z>aFr(W_{}7i22Jr>Z9gQI)IOivT z!v*hgR|QDU4^3%2rnNkx=;a+}y5uYdjMIsvo2X-gQc~H0FMxlJ3A{M$m47=A#$P(m zKjZ|6V1F|m*Z+1-ko?EeIr$%#&Y%RkfdcjqaaAdrS&=}w31OCAQo5am0Ho~1s(5AY zWd#XJ-{yqdCgu6K{~Q-Q?R&@kF5$1W?Jx6!|Eqgx_U-(mZ}}Mej{Nt4d1dFYf3jVb z9HWHah<|0fj5_~+BnHxVmsbdjF1js*&KczbO8|d@9A9vG`wjV5sm33W|I?8{75ti@ zbJ6P4#Ubvq71W>j{xLGZ`^%5&|3YTaNW^&kyJ@I@Ksgw>>KN-;TI%Z=Sp0Kb5Qy-% zyN2VByz5_%EY%aXrHduuLkgio1;cRS`~fH0I75kiVv1%0MT}W$`FJ4#VZt;j!bHfe zsaA;qNQ9*0MWN8a?Zsj0^prsp%yKG7|; zVT2h!(xE6wvBU*ssM=^)4Nc|{e;5QuCyABdZXmS-wIwiebjr9rI$6`8rQ(Dj8Xax# zja#zc8m4M2mx=IJrW7Pe2k07n;s}_$M;Kd0S{GD)Y?d+mbL%_9BwT~%#X zwY{VD&_G@4=7=4?r^mw#)$VXfID$4JEhAE7vwB zpz;%BgDa4^na-IwwP)&(XIH574MZRi`0$!mU{gPe_l^NDKqp#yt7^armNJ1@>A^MU zhsNZ#N2G=1m_Ic%ZWxhT8lj z)N9Y!KBhVZ2lXBVr;n=odqY!+ING~>Rg}`5#{e|51s#%-Dh>$!U#!+{CU6OROFfrW zq^^0czU5p@nzb%5$VmfO~cOzUe4!=o`TdDBEjsI0ue zAodESmJ7W1uO=R>7h-Iel@g>WT}Qt2`*+8h$$ntt?8lh)dM|=({7q=`s|$Y8Q80sd zE`%#(`WDV@W4=X!Tf64eXZ$4QjHl|zaqJ8Tr(RZ^8IPAhsxS0l++>ml`lFbap_4x7 z4nI*v+5d2~K@4L523uVPG@f{?fzv)3L+2-IB^++Rz7VUM_C1_uZ+f_2sIKugK@nw{ zDTMt^^}60PNJ^Pgx$e~vRP*+RpDx8rA(r6+(qY?vrSisE6Kkz01f*HSKoDN3h5Bm+=s{WC}^2=_CB@ODN0j zBJtdcqk;f=2Mu1b^dux9VHuHd@_5~WE@zEBmBToadsha#FFe*AJwPhfjCsc0iaiGn zEQ8HF{`k^-BOj}|; zxa(M28S_V_eT9+dYM)razlt*iawEFTYjQIHHEJNiC zB@PpHkf1SE9?x>X^vRylR}tazSit<4JeCQ}PgSuV1QM-jCkN93`l#g+_mB3P?)l}g zscX66wuwY0PmOI#79$|Ee1Xr=B<{siDMeqb^YyI|z4h7Tx>_nQaQFS3DQy~R>2GlHI z^C@SA5_gVqj~qyct;qosf-nCbI=aWn1;``Ap_!joBBfQpHCI!_GUNORaMj#=<2i7< zzVi6)pj?)oeM10H&y+ty|9CZk!DK9;$D=K( zcXeo>iAxas1~ElxgL8@F$=#4S1KGGKPGD@cyKY8X&fZd}U_M`7Jxd0zi#x0?8$ZDwj+aJJ)0V8az3zxg@jU7>upuJS<1_hSh_qF zoxo%qUktAKLf-Wd&?iY!xI$)FmWLG#5>-HafZppNs*glEK*701=B+22Ha_~B^-fe-cag5gJZ)XWOF53Li2#BG&hsQDhw&#A00%kwvL>p2_vh< z6y!E|N&HsOuX}jI1HY&!auZcHj)81zdtr3DRTwmWmdp+nhQ;?aB)VMtfLrot$M9j* zgzH>cwr#aII{1q*zto)*N?pjS$9P3v2|~5UcaLd#u=_=kXbYe_F&)2;HZ8h?zR)DZ zreCuWSKHwMeAOo{=DzcfuQ#;|Q)~MGdJT}#+bwZ6-5yw5GhbreWSfakLER}%jnKco zc(E*b4}cbQhsCZQU0f@3`57HtGi8KQlS9cGc-7Cvew8u^=#$92l1c+@WdgNZz1HwJ zAVr(2?`At-o$Q>Hi_azjCw`<3VJOLpJROqU@9tAxV`KD&y&z_~jUpc^aYkCPPL8XS zh8a>A>rRn~qN^PCs#AxYG#XE5*99gHl~4NzcPeev=DaB(`%j^Q^cls$94mCj-*$d9 z*M4rWXcly&R-V-w>$83JM1Z~IYTtv}Trz@O>`IAoo3+Ko6?CU8(`B}9WOr#1fmVDv-g&Eve}a>R zk+WHl#7wCwBQgEeGUOU=XUuR?r#M0&6JvbGWKWs5L60jF!R!jC0Io)_ZG@!0`B$C3 za-`mC_O-Yb+KFv1t}`$b&BSH?<`VdQts$s{X~Q5WI9uKq#@x1gK{QlrdmdrsykdCQ8>3 zZ-`T&pA)f<`zU<#+-B;`PY1E>}fSb$lpzSy@QYtO)Op zt(oK(xXQ!ktyk?pSI$o51eI;A)M6PAs#UxLKp` z4K~AVuQx5GAdwxHA1B$SIVz+=*3q+BUVVc^^Ys=wJS(2Aj&}Pkmz!L2HHdcpR~aoLy^n+45T44vo1oS#=kR$85h}(WFVJyBTj8pp(5LA8?oC_o5nL zD%$o_64k*8H*ps5Yr>^a*ZU&8JaOH<`3+Qc01~lLOt)gSwd|L^7^Yhz%yM$0QaZt_ z;~6r7oe?|JeV~9_0JxOdf?lFc4AJ4dpohsb7mq@rf=%eroOMil@}szym@y`;K&4u< zS}<=W286kxjg;X9s3hE883-o;d&zr)Gz}~3e&XV!TdK#yV9GY>{IB3nCq0>B1D|lAy#+K;gl5afVyjdxLFi?vlKCXg7@s{&*`sAy6NK%(fD4kmv#ewihW(++;w`-Z^FirZ0F16W{+pW z4x#80z+pRe2e-7OGv-$1>E~?F>GKBSSK_iyz+4X=(q@#WGMWzBsU21ALY}^dMfb3&MzrW0^f&)K}sMFfNo=U(-Qwu37DQwE1NhTYO z6TAd7Wp~=ct}2&5g6R(vWznZE2W0jkJFwQ;q;`^I)4K z1XRH%Xc%RdDE96Z82sWr>W?{Px$j#z7!0u8wmFocov7Z|X#qL$GLUkb343=k6Z^pr zl*pu)0{$QttP?V&srhvCDlUOR!sJ=tUwg5r+M%2qtd^!O^i)644XVa z*6Plcw*v!qSu`lfVx4QN6yVO$354o$Gj$hD=*nqFo%MkI>|_e+IR!7InmDc((&Fko z;-O80FZlxc=KxRhP^tKL0P!#10RN`|;vcd*z`wtA%Isgmhd*9I_5bw^1w%uekV8ZN znB9p0K={V#e?%%V1Ss@~fuzXw(0CZ*`!dM1p1gaS+C!`*sa<;30(T8kwh-4A|FxVAu^Yjp z$?1OFA9>85U`CiXoe>=Fcl01uS%P$BQxlmL@Qv{7_vO@Sr~4sfCdK-zQ<7c{g{!Wr zYDTn@M`LdNpx1l2zC1NIm#jK$l26aDu2%k=(sx(abD&kX`?&e99#%Px{cMo_x=FsIq(#f|Bvr$qt$kUs{RZ{6vZjp680#w^a+wn%sxdeFDr~;5dgsaG(J4Xw0RnZMQA1YnIPK=kB z=rPIERNUecG-@UOc*4@SRnkDR+VbXO8#61-4Mvd#z82dO^_Ob1jvo_lvA0Or9~C@M z`!Kgm{XtU5sKl#_y$Cmw*G&Kga? zB1*!1M6q0=5G#8_D!B5tDB%G})i2G&U0XP-7d?8cf0V8^gq`vT*M`O29khK<@J?-) z4wANtK6ChmkN+Ls=EKLSqOHSK8~w-%p3h5zhcgxwA<8AY72xzicq2Ht*a4)FEvs>A zYEt4;P|6XbFvmpy`ktVjqE}%Fa1} zNJ>P2WiL5CjsIbW3P0L^OulX()?^66v`V-5Cfb$w>h>=0iM`HN&|u5Rcw> zWZsk??@)Y<^N!M%uol0sAljiu7)pPE_PMh{>=)3kqR6rTJl0X!OF8BONry6AD5?e0 zF$>PcB<)xyB%mml$UjsbKqUe2kFmzNn548#JaG6(0Yw17CP;yLyUyD>FAhFoQp}S z=YhqQ!%lPxQb8gybI9_*1}x`dk}8ZBI;HQ;M>t5wz~>O^PxLmPF#ME*qzPO(lq6*n zUO7&@y7bRd8Wc42?i7$E5abwuIqaX6c5q|ARz#f1J)p5A22Xv7sc7M z5Osq$0REUQ?C{aYZInF$gO6meDaYp;>cwUX5rLmtuy=-)AVeK>^fVAL1PKLuTwEvx zgb-f`f-3^%7i>rI2!uX7rimy5<$-M^>w~N{WaJ5k!zgu#ChdLY2O@5a;ve5&E3i=}L5m WAd)B+SujzszZ`7Lw`2b 0.05 * image.max()) & (x >= x_c) - x0, y0 = (np.sum(c[iso] * image[iso]) / image[iso].sum() for c in (x, y)) - assert np.hypot(x0 - x_c, y0 - y_c) > 1.0 - - hlr = sigma * np.sqrt(2 * np.log(2)) - xw, yw, flags = wp.windowed_positions(image, [x0], [y0], [hlr]) - assert flags[0] == 0 - assert abs(xw[0] - x_c) < 1e-5 and abs(yw[0] - y_c) < 1e-5 - - -def test_neighbour_footprint_is_masked_by_its_mirror(): - """A neighbour's footprint does not pull the centroid. - - A neighbour ten times brighter, 8 px away, captures the window when - nothing masks it. With its footprint in the map, those pixels take the - galaxy's mirror image and the centroid stays within 0.02 px of the - isolated galaxy's (the rest is the neighbour's wings outside its - footprint). - """ - x_c, y_c, sigma = 30.0, 30.6, 2.0 - galaxy = _gaussian((61, 61), x_c, y_c, sigma) - neighbour = _gaussian((61, 61), x_c + 8.0, y_c + 1.0, 1.5, flux=20000.0) - y, x = np.mgrid[1:62, 1:62] - seg = np.where(np.hypot(x - x_c - 8.0, y - y_c - 1.0) < 5.5, 2, 0) - seg[np.hypot(x - x_c, y - y_c) < 3.0] = 1 - hlr = sigma * np.sqrt(2 * np.log(2)) - start = ([x_c + 0.5], [y_c], [hlr]) - - alone = wp.windowed_positions(galaxy, *start) - masked = wp.windowed_positions(galaxy + neighbour, *start, seg=seg, - number=[1]) - unmasked = wp.windowed_positions(galaxy + neighbour, *start) - assert np.hypot(masked[0][0] - alone[0][0], - masked[1][0] - alone[1][0]) < 0.02 - assert unmasked[0][0] - alone[0][0] > 1.0 - - -def test_unmeasurable_and_negative_objects_keep_the_start(): - """Off-image, NaN and negative-flux objects keep the input position.""" - image = _gaussian((41, 41), 20.0, 20.0, 2.0) - x0 = np.array([20.2, 60.0, np.nan]) - y0 = np.array([20.1, 20.0, 20.0]) - xw, yw, flags = wp.windowed_positions(image, x0, y0, [2.35, 2.35, 2.35]) - assert flags[0] == 0 - npt_equal = np.testing.assert_array_equal - npt_equal(flags[1:], wp.FLAGS_WIN_UNMEASURED) - npt_equal(xw[1:], x0[1:]) - - xw, yw, flags = wp.windowed_positions(-image, [20.2], [20.1], [2.35]) - assert flags[0] & wp.FLAGS_WIN_NEGATIVE_FLUX - assert (xw[0], yw[0]) == (20.2, 20.1) - - -def test_negative_flux_flag_is_sextractors(): - """A centred negative source is FLAGS_WIN 4 exactly, as in SExtractor. - - SExtractor divides the (negative) second-moment sums by the negative - flux, so the moments come out positive and only bit 4 is set. - """ - image = -_gaussian((41, 41), 20.0, 20.0, 2.0) - _, _, flags = wp.windowed_positions(image, [20.0], [20.0], [2.35]) - assert flags[0] == wp.FLAGS_WIN_NEGATIVE_FLUX - - -HLR = 2.0 * np.sqrt(2 * np.log(2)) - - -def test_invalid_stripe_is_mirrored(): - """A no-data stripe through the window takes the mirror image. - - A Gaussian at (31, 31) with columns 33-35 holding no data: what the - stripe holds is ignored, and a centroid at the centre stays there. Read - as signal, the zero stripe drags it 0.6 px left. (From other starts - SExtractor's integer mirror index can settle up to half a pixel off, at - 31.44 from (31.3, 30.8); this is SExtractor's behaviour too.) - """ - image = _gaussian((61, 61), 31.0, 31.0, 2.0) - valid = np.ones(image.shape, bool) - valid[:, 32:35] = False - zero = np.where(valid, image, 0.0) - garbage = np.where(valid, image, 1e4) - - xw, yw, flags = wp.windowed_positions(zero, [31.0], [31.0], [HLR], - valid=valid) - assert flags[0] == 0 - assert abs(xw[0] - 31.0) < 1e-6 and abs(yw[0] - 31.0) < 1e-6 - for start in (([31.0], [31.0]), ([31.3], [30.8])): - a = wp.windowed_positions(zero, *start, [HLR], valid=valid) - b = wp.windowed_positions(garbage, *start, [HLR], valid=valid) - np.testing.assert_array_equal(a[0], b[0]) - np.testing.assert_array_equal(a[1], b[1]) - xw, _, _ = wp.windowed_positions(zero, [31.0], [31.0], [HLR]) - assert xw[0] < 30.8 - - -def test_invalid_and_off_image_mirrors_count_as_zero(): - """A mirror off the image, or itself invalid, contributes 0. - - Invalid pixels hold garbage; the result equals that of the same image - with those pixels set to 0 and all valid, near the left edge (mirrors - off the image) and with both a stripe and its mirror invalid. - """ - rng = np.random.default_rng(5) - for x_c, cols in ((4.0, np.r_[8:11]), (31.0, np.r_[26:29, 32:35])): - image = _gaussian((61, 61), x_c, 31.0, 2.0) - valid = np.ones(image.shape, bool) - valid[:, cols] = False - zeroed = np.where(valid, image, 0.0) - garbage = np.where(valid, image, rng.uniform(1e3, 1e4, image.shape)) - start = ([x_c + 0.1], [31.2], [HLR]) - got = wp.windowed_positions(garbage, *start, valid=valid) - want = wp.windowed_positions(zeroed, *start) - np.testing.assert_allclose(got[:2], want[:2], rtol=0, atol=1e-9) - assert got[2][0] == want[2][0] - - -def test_nan_outside_the_aperture_is_ignored(): - """A NaN that enters the stamp square after the first step changes nothing. - - From (30.2, 30.2) the square first spans 0-based columns 18-40; once - the centre reaches (31, 31) it spans 19-41, and a NaN at column 41, - outside the aperture, must not reach the sums. - """ - image = _gaussian((61, 61), 31.0, 31.0, 2.0) - clean = wp.windowed_positions(image, [30.2], [30.2], [HLR]) - image[30, 41] = np.nan - got = wp.windowed_positions(image, [30.2], [30.2], [HLR]) - assert got[2][0] == 0 - np.testing.assert_allclose(got[:2], clean[:2], rtol=0, atol=1e-12) - - -def test_nan_inside_the_aperture_is_mirrored(): - """A NaN pixel on the galaxy is invalid and takes its mirror image.""" - image = _gaussian((61, 61), 31.0, 31.0, 2.0) - image[31, 32] = np.nan - xw, yw, flags = wp.windowed_positions(image, [30.6], [31.3], [HLR]) - assert flags[0] == 0 - assert abs(xw[0] - 31.0) < 1e-3 and abs(yw[0] - 31.0) < 1e-3 - - -def test_wild_flux_radius_is_flagged_without_allocating(): - """FLUX_RADIUS <= 0 or NaN is an empty aperture; a huge one is skipped. - - DR6 carries FLUX_RADIUS down to -1.2e6; taken as a size, it built an - 84k-px box. Non-positive and NaN radii give FLAGS_WIN 4, radii whose - window exceeds MAX_HALF give 16, both at the barycentre, and the - measurable object alongside is unaffected. - """ - import tracemalloc - - image = _gaussian((101, 101), 51.0, 51.0, 2.0) - fr = [-12402.6, 0.0, np.nan, 1.18e6, HLR] - x0 = [50.5, 50.5, 50.5, 50.5, 50.6] - tracemalloc.start() - xw, yw, flags = wp.windowed_positions(image, x0, [51.0] * 5, fr) - peak = tracemalloc.get_traced_memory()[1] - tracemalloc.stop() - assert peak < 200e6 - np.testing.assert_array_equal( - flags, [4, 4, 4, wp.FLAGS_WIN_UNMEASURED, 0]) - np.testing.assert_array_equal(xw[:4], x0[:4]) - assert abs(xw[4] - 51.0) < 1e-5 - - -def test_box_is_clamped_to_a_small_image(): - """A window wider than the image measures the image, not a huge box.""" - image = _gaussian((9, 9), 5.0, 5.0, 1.0) - xw, yw, flags = wp.windowed_positions(image, [5.2], [4.9], [60.0]) - assert flags[0] & wp.FLAGS_WIN_UNMEASURED == 0 - assert np.isfinite(xw[0]) - - -def test_background_ignores_nan_pixels(): - """A NaN in a flat sky-3 mesh leaves the background at 3.""" - image = np.full((200, 200), 3.0) - image[100, 100] = np.nan - back = wp.sextractor_background(image, back_size=256, filter_size=3, - good=image != 0) - np.testing.assert_allclose(back, 3.0, atol=1e-6) - - -def _fixture_objects(): - with np.load(FIXTURE) as data: - names = sorted({k.rsplit("_", 1)[0] for k in data.files - if k.endswith("_image")}) - return {n: {k[len(n) + 1:]: data[k] for k in data.files - if k.startswith(n + "_")} for n in names} - - -@pytest.mark.parametrize("name", sorted(_fixture_objects())) -def test_matches_sextractor_on_real_objects(name): - """SExtractor 2.25's XWIN_IMAGE and FLAGS_WIN, object by object. - - The objects are a bright and a faint isolated galaxy, two whose window - reaches a neighbour's footprint (masked_neighbour*: unmasked, they are - 0.07 and 0.6 px off), one SExtractor falls back to the barycentre for - (FLAGS_WIN 1) and one that has not converged after 16 steps. A fallback - agrees only to the float32 rounding of the catalogue's X_IMAGE. - """ - obj = _fixture_objects()[name] - x0, y0, fr = obj["start"] - xw, yw, flags = wp.windowed_positions( - obj["image"], [x0], [y0], [fr], seg=obj["seg"], number=[1], - cxx=obj["ellipse"][:1], cyy=obj["ellipse"][1:2], - cxy=obj["ellipse"][2:], - ) - expected = int(obj["flags_win"]) - assert flags[0] & 15 == expected - tol = 1e-3 if expected else 1e-5 - assert abs(xw[0] - obj["expected"][0]) < tol - assert abs(yw[0] - obj["expected"][1]) < tol - if name == "not_converged": - assert flags[0] & wp.FLAGS_WIN_NOT_CONVERGED - - -def test_background_recovers_a_smooth_sky(): - """A tilted sky plus sources comes back to well under the noise.""" - rng = np.random.default_rng(3) - ny, nx = 1200, 1500 - y, x = np.mgrid[0:ny, 0:nx] - sky = 5.0 + 1e-3 * x - 5e-4 * y - image = sky + rng.normal(0, 1.0, (ny, nx)) - for _ in range(100): - image += _gaussian((ny, nx), *rng.uniform(1, [nx, ny]), 2.0, - flux=rng.uniform(50, 500)) - back = wp.sextractor_background(image, back_size=256, filter_size=3) - assert np.abs(back - sky)[100:-100, 100:-100].max() < 0.1 diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini index 19e9d9d87..ab9d04742 100644 --- a/workflow/config/cfis/config_tile_Uc.ini +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -88,10 +88,8 @@ SEGMENTATION = True # @sc [decision:detection.epoch_membership_ccd_bounds] MAKE_POST_PROCESS = True -# World coordinate keywords: the windowed position read_ext_sexcat measures. -# Format: KEY_X,KEY_Y -# @sc [decision:preparation.object_position_columns] -WORLD_POSITION = XWIN_WORLD,YWIN_WORLD +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = ALPHA_J2000,DELTA_J2000 # Number of pixels in x,y of a CCD. Format: Nx,Ny # @sc [decision:detection.epoch_membership_ccd_bounds] diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 93a0c5f7a..cc5feaabf 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -2,10 +2,6 @@ # @sc [decision:preparation.object_position_columns] XWIN_WORLD YWIN_WORLD -# FLAGS_WIN: how the windowed position was obtained. 1-8 are SExtractor's -# reasons to fall back to the isophotal barycentre; under unions_catalogue -# also 16 (not measurable, barycentre kept) and 32 (not converged in 16 steps). -FLAGS_WIN # tile ID, for plot of tile-dependent additive bias. # Can maybe be removed. diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 96f806d05..9db095b66 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -15,10 +15,6 @@ # coordinates XWIN_WORLD YWIN_WORLD -# FLAGS_WIN: how the windowed position was obtained. 1-8 are SExtractor's -# reasons to fall back to the isophotal barycentre; under unions_catalogue -# also 16 (not measurable, barycentre kept) and 32 (not converged in 16 steps). -FLAGS_WIN # tile ID, for plot of tile-dependent additive bias. TILE_ID From a57e8efcf2c9638c659115a1c70072642f25871a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:01:40 +0200 Subject: [PATCH 29/44] Config values expand ${VAR:-default} A value with ${VAR:-default} expands to default when VAR is unset or empty, as in the shell; $VAR and ${VAR} stay strict. The same change as feat/uberseg-seg-vignet's, so the two merge cleanly. Co-Authored-By: Claude Opus 5.5 --- .../read_ext_sexcat_package/__init__.py | 1 - .../read_ext_sexcat.py | 358 --------------- .../modules/read_ext_sexcat_runner.py | 85 ---- src/shapepipe/pipeline/config.py | 30 +- tests/module/test_final_cat_columns.py | 141 ------ tests/module/test_read_ext_sexcat.py | 424 ------------------ workflow/config/cfis/config_tile_Uc.ini | 97 ---- workflow/scripts/merge_final_cat.py | 48 +- 8 files changed, 25 insertions(+), 1159 deletions(-) delete mode 100644 src/shapepipe/modules/read_ext_sexcat_package/__init__.py delete mode 100644 src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py delete mode 100644 src/shapepipe/modules/read_ext_sexcat_runner.py delete mode 100644 tests/module/test_final_cat_columns.py delete mode 100644 tests/module/test_read_ext_sexcat.py delete mode 100644 workflow/config/cfis/config_tile_Uc.ini diff --git a/src/shapepipe/modules/read_ext_sexcat_package/__init__.py b/src/shapepipe/modules/read_ext_sexcat_package/__init__.py deleted file mode 100644 index cc65bc9fd..000000000 --- a/src/shapepipe/modules/read_ext_sexcat_package/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""READ EXTERNAL SEXCAT PACKAGE.""" diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py deleted file mode 100644 index de9af6511..000000000 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ /dev/null @@ -1,358 +0,0 @@ -"""READ EXTERNAL SEXCAT. - -Convert an ASCII SExtractor-format catalogue to FITS-LDAC for use in -ShapePipe tile processing. - -:Author: Martin Kilbinger - -""" - -import numpy as np -from astropy.io import ascii as asc -from astropy.io import fits - - -def _build_ldac_imhead(img_header): - """Build LDAC_IMHEAD extension from a FITS image header. - - The LDAC_IMHEAD binary table has one row and one column containing - the image header cards as an array of 80-character strings. The - HISTORY cards are the ones used by ``make_post_process`` to identify - the single exposures that contributed to the tile. - - Parameters - ---------- - img_header : astropy.io.fits.Header - Header of the tile image FITS file - - Returns - ------- - astropy.io.fits.BinTableHDU - LDAC_IMHEAD extension - - """ - cards = [str(card).ljust(80)[:80] for card in img_header.cards] - n_cards = len(cards) - header_str = "".join(cards) - - arr = np.array([header_str.encode()], dtype=f"S{len(header_str)}") - col = fits.Column( - name="Field Header Card", - format=f"{len(header_str)}A", - array=arr, - ) - hdu = fits.BinTableHDU.from_columns([col]) - hdu.header["TDIM1"] = f"(80,{n_cards})" - hdu.name = "LDAC_IMHEAD" - return hdu - - -# SExtractor's VIGNET value for pixels that are not the object's: off the -# image and on a neighbour's segmentation footprint. ngmix flags these pixels -# (tile VIGNET == -1e30) and noise-fills them at zero weight. -BIG = -1e30 - -# The label of a footprint no catalogue object claims in the relabelled -# segmentation map. Negative, so it never collides with a NUMBER; VIGNET -# marking only asks "self or not self", so neighbours need no identity. -NEIGHBOUR_LABEL = -1 - -# Radius, in pixels, of the disc of its own NUMBER painted for an object with -# no footprint of its own (on sky, or on a footprint claimed by another). -FALLBACK_RADIUS = 3 - - -def _centre_pixels(x_pos, y_pos): - """0-based (column, row) of the pixel holding each 1-based position. - - @sc [decision:preparation.stamp_positioning_and_padding] - """ - col = np.rint(np.asarray(x_pos, dtype=float)).astype(np.int64) - 1 - row = np.rint(np.asarray(y_pos, dtype=float)).astype(np.int64) - 1 - return col, row - - -def relabel_seg(seg, number, x_image, y_image, fallback_radius=FALLBACK_RADIUS): - """Relabel a segmentation map into a catalogue's NUMBER. - - Each object claims, in NUMBER order and first come first served, the - footprint its centre pixel falls in; that footprint becomes its NUMBER. - Footprints nobody claims become ``NEIGHBOUR_LABEL``; sky stays 0. An - object on sky or on an already-claimed footprint gets a disc of radius - ``fallback_radius`` of its NUMBER, and every object on the image ends - holding its own centre pixel (the lower NUMBER wins a shared pixel). - - Parameters - ---------- - seg : numpy.ndarray - Segmentation map, 0 for sky and one positive label per footprint - number : array_like - Catalogue ``NUMBER`` - x_image, y_image : array_like - 1-based pixel positions on the grid of ``seg`` - fallback_radius : int, optional - Radius of the disc painted for an object without a footprint - - Returns - ------- - numpy.ndarray - Relabelled map, ``int32`` - dict - Counts of the claim outcomes ``matched``, ``unclaimed`` (on sky), - ``shared`` and ``off_image``, which partition the catalogue, plus - ``shared_pixel`` (objects rounding to a lower NUMBER's pixel) - - @sc [decision:detection.catalogue_neighbour_marking] - """ - number = np.asarray(number) - col, row = _centre_pixels(x_image, y_image) - n_row, n_col = seg.shape - inside = (col >= 0) & (col < n_col) & (row >= 0) & (row < n_row) - order = np.argsort(number, kind="stable") - - counts = dict(matched=0, unclaimed=0, shared=0, off_image=0, - shared_pixel=0) - table = np.full(max(int(seg.max()), 0) + 1, NEIGHBOUR_LABEL, np.int32) - table[0] = 0 - claimed, fallback = set(), [] - for i in order: - if not inside[i]: - counts["off_image"] += 1 - continue - label = int(seg[row[i], col[i]]) - if label == 0 or label in claimed: - counts["unclaimed" if label == 0 else "shared"] += 1 - fallback.append(i) - else: - claimed.add(label) - table[label] = number[i] - counts["matched"] += 1 - out = table[np.clip(seg, 0, None)] - - r = np.arange(-fallback_radius, fallback_radius + 1) - disc = np.argwhere(np.add.outer(r**2, r**2) <= fallback_radius**2) - disc -= fallback_radius - for i in fallback: - out[np.clip(row[i] + disc[:, 0], 0, n_row - 1), - np.clip(col[i] + disc[:, 1], 0, n_col - 1)] = number[i] - - # Centres last, so no disc can take an object's centre away. - taken = set() - for i in order[inside[order]]: - pixel = (row[i], col[i]) - if pixel in taken: - counts["shared_pixel"] += 1 - continue - taken.add(pixel) - out[pixel] = number[i] - return out, counts - - -def _extract_vignets(image_data, x_pos, y_pos, stamp_size, seg=None, - number=None): - """Extract postage stamps from a tile image array. - - For each object position, a ``stamp_size x stamp_size`` cutout is - extracted, centred on the pixel holding the position. Pixels off the - image are set to ``BIG``, as SExtractor does. With a segmentation map in - the catalogue's numbering (:func:`relabel_seg`), pixels on any footprint - other than the object's own are set to ``BIG`` too, which is how - SExtractor's VIGNET marks neighbours. - - Parameters - ---------- - image_data : numpy.ndarray - 2-D tile image array, shape ``(ny, nx)`` - x_pos : array_like - X pixel positions, 1-based (SExtractor convention) - y_pos : array_like - Y pixel positions, 1-based (SExtractor convention) - stamp_size : int - Side length of the square postage stamp (should be odd) - seg : numpy.ndarray, optional - Segmentation map on the image grid, labelled with ``number`` - number : array_like, optional - Catalogue ``NUMBER``, required with ``seg`` - - Returns - ------- - numpy.ndarray - Array of shape ``(n_obj, stamp_size, stamp_size)``, dtype float32 - - @sc [decision:detection.catalogue_neighbour_marking] - """ - ny, nx = image_data.shape - half = stamp_size // 2 - col, row = _centre_pixels(x_pos, y_pos) - vignets = np.full((len(col), stamp_size, stamp_size), BIG, np.float32) - seg_stamp = np.zeros((stamp_size, stamp_size), np.int32) - - for i, (xi, yi) in enumerate(zip(col, row)): - x0, y0 = xi - half, yi - half - xc0, xc1 = max(0, x0), min(nx, x0 + stamp_size) - yc0, yc1 = max(0, y0), min(ny, y0 + stamp_size) - if xc0 >= xc1 or yc0 >= yc1: - continue - inner = np.s_[yc0 - y0:yc1 - y0, xc0 - x0:xc1 - x0] - vignets[i][inner] = image_data[yc0:yc1, xc0:xc1] - if seg is not None: - seg_stamp[:] = 0 - seg_stamp[inner] = seg[yc0:yc1, xc0:xc1] - vignets[i][(seg_stamp != 0) & (seg_stamp != number[i])] = BIG - - return vignets - - -def _build_ldac_objects(cat_data, vignets): - """Build LDAC_OBJECTS extension from an astropy table. - - Parameters - ---------- - cat_data : astropy.table.Table - Catalogue data read from the ASCII SExtractor file - vignets : numpy.ndarray - Array of shape ``(n_obj, stamp_size, stamp_size)`` - - Returns - ------- - astropy.io.fits.BinTableHDU - LDAC_OBJECTS extension - - """ - stamp_size = vignets.shape[1] - n_pix = stamp_size * stamp_size - - fits_cols = [] - for colname in cat_data.colnames: - arr = np.array(cat_data[colname]) - kind = arr.dtype.kind - if kind in ("i", "u"): - fmt = "J" - elif kind == "f": - fmt = "E" if arr.dtype.itemsize <= 4 else "D" - else: - fmt = f"{arr.dtype.itemsize}A" - fits_cols.append(fits.Column(name=colname, format=fmt, array=arr)) - - _aliases = { - "XWIN_IMAGE": "X_IMAGE", - "YWIN_IMAGE": "Y_IMAGE", - "XWIN_WORLD": "ALPHA_J2000", - "YWIN_WORLD": "DELTA_J2000", - } - col_map = {c.name: c for c in fits_cols} - for alias, source in _aliases.items(): - if source in col_map and alias not in col_map: - src_col = col_map[source] - fits_cols.append( - fits.Column( - name=alias, - format=src_col.format, - array=np.array(cat_data[source]), - ) - ) - - vignet_col = fits.Column( - name="VIGNET", - format=f"{n_pix}E", - array=vignets.reshape(len(vignets), n_pix), - dim=f"({stamp_size},{stamp_size})", - ) - fits_cols.append(vignet_col) - - hdu = fits.BinTableHDU.from_columns(fits_cols) - hdu.name = "LDAC_OBJECTS" - return hdu - - -def make_ldac_from_ascii( - input_cat_path, - image_path, - output_cat_path, - stamp_size=51, - seg_path=None, - w_log=None, -): - """Convert an external ASCII catalogue to FITS-LDAC format. - - Reads an ASCII catalogue in SExtractor format (column definitions as - ``# N NAME ...`` comment lines followed by data rows), attaches the - tile image header as an ``LDAC_IMHEAD`` extension so that - ``make_post_process`` can recover the contributing single exposures, and - writes a standard FITS-LDAC file compatible with all downstream ShapePipe - modules. - - The input columns, ``NUMBER`` included, are copied unchanged; a - ``VIGNET`` column (postage stamps extracted from the tile image) is - added to ``LDAC_OBJECTS``. Given the catalogue's segmentation map, which - shares the tile's pixel grid, the map is relabelled to the catalogue's - ``NUMBER`` (:func:`relabel_seg`) and used to set neighbours' pixels in each ``VIGNET`` to ``BIG``, as - SExtractor does. - - Parameters - ---------- - input_cat_path : str - Path to input ASCII catalogue in SExtractor format - image_path : str - Path to tile image FITS file - output_cat_path : str - Path to the output FITS-LDAC catalogue - stamp_size : int, optional - Side length of the square postage stamp in pixels, default 51 - seg_path : str, optional - Path to the catalogue's segmentation map (FITS, compressed or not) - w_log : logging.Logger, optional - Pipeline logger - - """ - cat_data = asc.read(input_cat_path, format="sextractor") - n_obj = len(cat_data) - if w_log: - w_log.info(f"Read {n_obj} objects from {input_cat_path}") - - with fits.open(image_path) as hdul: - img_header = hdul[0].header - image_data = hdul[0].data.astype(np.float32) - - seg = None - if seg_path is not None: - with fits.open(seg_path) as hdul: - hdu = next(h for h in hdul if h.data is not None) - seg_raw = hdu.data - if seg_raw.shape != image_data.shape: - raise ValueError( - f"Segmentation map {seg_path} has shape {seg_raw.shape}, the" - + f" image {image_data.shape}; they must share one grid." - ) - seg, counts = relabel_seg( - seg_raw, cat_data["NUMBER"], cat_data["X_IMAGE"], - cat_data["Y_IMAGE"], - ) - del seg_raw - if w_log: - w_log.info( - f"Relabelled {seg_path} to NUMBER: " - + ", ".join(f"{k}={v}" for k, v in counts.items()) - ) - - if w_log: - w_log.info( - f"Extracting {stamp_size}x{stamp_size} vignets from {image_path}" - ) - vignets = _extract_vignets( - image_data, - cat_data["X_IMAGE"], - cat_data["Y_IMAGE"], - stamp_size, - seg=seg, - number=np.asarray(cat_data["NUMBER"]), - ) - - ldac_imhead = _build_ldac_imhead(img_header) - ldac_objects = _build_ldac_objects(cat_data, vignets) - - hdul_out = fits.HDUList([fits.PrimaryHDU(), ldac_imhead, ldac_objects]) - hdul_out.writeto(output_cat_path, overwrite=True) - - if w_log: - w_log.info(f"Written FITS-LDAC catalogue to {output_cat_path}") diff --git a/src/shapepipe/modules/read_ext_sexcat_runner.py b/src/shapepipe/modules/read_ext_sexcat_runner.py deleted file mode 100644 index 17e23163d..000000000 --- a/src/shapepipe/modules/read_ext_sexcat_runner.py +++ /dev/null @@ -1,85 +0,0 @@ -"""READ EXTERNAL SEXCAT RUNNER. - -Module runner for ``read_ext_sexcat``. - -:Author: Martin Kilbinger - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.read_ext_sexcat_package import read_ext_sexcat as rs -from shapepipe.modules.sextractor_package import sextractor_script as ss - - -@module_runner( - version="1.0", - input_module=["get_images_runner"], - file_pattern=["CFIS_cat", "CFIS_image"], - file_ext=[".cat", ".fits"], - depends=["numpy", "astropy"], -) -def read_ext_sexcat_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define the Read External SExtractor Catalogue Runner. - - Reads an external ASCII catalogue (SExtractor format), converts it to - a FITS-LDAC catalogue compatible with downstream ShapePipe modules. - The inputs are the catalogue and the tile image, then the catalogue's - segmentation map if SEGMENTATION = True, then the WCS log if - MAKE_POST_PROCESS = True. With the segmentation map, neighbours' pixels - in VIGNET are set to -1e30; the map itself is not written out; the - only output is the FITS-LDAC ``sexcat.fits``. MAKE_POST_PROCESS runs the - multi-epoch post-processing that adds per-exposure HDUs. - """ - cat_path, image_path, *extra_inputs = input_file_list - use_seg = config.has_option( - module_config_sec, "SEGMENTATION" - ) and config.getboolean(module_config_sec, "SEGMENTATION") - seg_path = extra_inputs.pop(0) if use_seg else None - - if config.has_option(module_config_sec, "SUFFIX"): - suffix = config.get(module_config_sec, "SUFFIX") - else: - suffix = "sexcat" - - output_path = f"{run_dirs['output']}/{suffix}{file_number_string}.fits" - - if config.has_option(module_config_sec, "VIGNET_SIZE"): - stamp_size = config.getint(module_config_sec, "VIGNET_SIZE") - else: - stamp_size = 51 - - w_log.info(f"Reading external catalogue: {cat_path}") - w_log.info(f"Reading image header from: {image_path}") - - rs.make_ldac_from_ascii( - cat_path, - image_path, - output_path, - stamp_size=stamp_size, - seg_path=seg_path, - w_log=w_log, - ) - - if config.getboolean(module_config_sec, "MAKE_POST_PROCESS"): - # The WCS log is supplied as a positional input (via FILE_PATTERN) - # when post-processing is enabled, not by the decorator default. - if not extra_inputs: - raise ValueError( - "MAKE_POST_PROCESS requires the WCS log file as the last" - + " input; add 'log_exp_headers' to FILE_PATTERN and" - + f" FILE_EXT in the [{module_config_sec}] config section." - ) - f_wcs_path = extra_inputs[0] - pos_params = config.getlist(module_config_sec, "WORLD_POSITION") - ccd_size = config.getlist(module_config_sec, "CCD_SIZE") - w_log.info("Running post-processing") - ss.make_post_process(output_path, f_wcs_path, pos_params, ccd_size) - - return None, None diff --git a/src/shapepipe/pipeline/config.py b/src/shapepipe/pipeline/config.py index 4e809316b..64d2e0592 100644 --- a/src/shapepipe/pipeline/config.py +++ b/src/shapepipe/pipeline/config.py @@ -11,18 +11,24 @@ from configparser import ConfigParser +# ``$VAR``, ``${VAR}`` or ``${VAR:-default}``. +_VAR_RE = re.compile(r"\$(?:\{(\w+)(?::-([^}]*))?\}|(\w+))") + + def _expandvars_strict(value): """Expand Environment Variables Strictly. Expand environment variables in ``value``, raising an error if any referenced variable is unset instead of silently leaving the literal - ``$VAR`` string in place. + ``$VAR`` string in place. ``${VAR:-default}`` expands to ``default`` + when ``VAR`` is unset or empty, as in the shell. Each value is inserted + as is, not expanded again. Parameters ---------- value : str - Configuration value possibly containing ``$VAR`` or ``${VAR}`` - references + Configuration value possibly containing ``$VAR``, ``${VAR}`` or + ``${VAR:-default}`` references Returns ------- @@ -32,11 +38,23 @@ def _expandvars_strict(value): Raises ------ ValueError - If a referenced environment variable is not set + If a referenced environment variable without a default is not set """ - expanded = os.path.expandvars(value) - unset = re.findall(r"\$\{?(\w+)\}?", expanded) + unset = [] + + def substitute(match): + braced, default, bare = match.groups() + name = braced or bare + env = os.environ.get(name) + if default is not None: + return env or default + if env is None: + unset.append(name) + return match.group(0) + return env + + expanded = _VAR_RE.sub(substitute, value) if unset: raise ValueError( f"Environment variable(s) {', '.join(sorted(set(unset)))} " diff --git a/tests/module/test_final_cat_columns.py b/tests/module/test_final_cat_columns.py deleted file mode 100644 index 51ea46772..000000000 --- a/tests/module/test_final_cat_columns.py +++ /dev/null @@ -1,141 +0,0 @@ -"""The final-catalogue columns each tile_detection mode requests. - -``final_cat_merge`` asks every tile catalogue for the columns of -``workflow/config/cfis/final_cat.param`` and stops on any one missing. That -list is written against SExtractor-mode tiles; under ``tile_detection: -unions_catalogue`` the detection columns are whatever the UNIONS per-tile -catalogue carries, copied through ``read_ext_sexcat``. So the catalogue-mode -request is the param list less ``merge_final_cat.SEXTRACTOR_ONLY_COLUMNS``. - -Here the catalogue-mode detection columns are not written down but derived: -the converter runs on a catalogue with the UNIONS DR6 header (copied verbatim -from ``vos:cfis/tiles_DR6/CFIS.202.301.r.cat``) and its output is what a tile -can carry. The requested SExtractor columns are those of the tile SExtractor -parameter file. Every other requested column comes from stages that run the -same way in both modes (post-processing, ngmix, make_cat), so the detection -columns are the whole of the difference between them. -""" - -import importlib.util -import re -import sys -from pathlib import Path - -import numpy as np -import pytest -from astropy.io import fits - -from shapepipe.modules.read_ext_sexcat_package import read_ext_sexcat as rs - -REPO_ROOT = Path(__file__).resolve().parents[2] -SCRIPTS = REPO_ROOT / "workflow" / "scripts" -CFIS = REPO_ROOT / "workflow" / "config" / "cfis" - -# The header of a UNIONS DR6 per-tile catalogue and its first object, moved to -# pixel (4, 4) so that its stamp falls on the small test image. -DR6_CATALOGUE = """\ -# 1 NUMBER Running object number -# 2 X_IMAGE Object position along x [pixel] -# 3 Y_IMAGE Object position along y [pixel] -# 4 ALPHA_J2000 Right ascension of barycenter (J2000) [deg] -# 5 DELTA_J2000 Declination of barycenter (J2000) [deg] -# 6 MAG_AUTO Kron-like elliptical aperture magnitude [mag] -# 7 MAGERR_AUTO RMS error for AUTO magnitude [mag] -# 8 MAG_BEST Best of MAG_AUTO and MAG_ISOCOR [mag] -# 9 MAGERR_BEST RMS error for MAG_BEST [mag] -# 10 MAG_APER Fixed aperture magnitude vector [mag] -# 11 MAGERR_APER RMS error vector for fixed aperture mag. [mag] -# 12 A_WORLD Profile RMS along major axis (world units) [deg] -# 13 ERRA_WORLD World RMS position error along major axis [deg] -# 14 B_WORLD Profile RMS along minor axis (world units) [deg] -# 15 ERRB_WORLD World RMS position error along minor axis [deg] -# 16 THETA_J2000 Position angle (east of north) (J2000) [deg] -# 17 ERRTHETA_J2000 J2000 error ellipse pos. angle (east of north) [deg] -# 18 ISOAREA_IMAGE Isophotal area above Analysis threshold [pixel**2] -# 19 MU_MAX Peak surface brightness above background [mag * arcsec**(-2)] -# 20 FLUX_RADIUS Fraction-of-light radii [pixel] -# 21 FLAGS Extraction flags - 1 4.0000 4.0000 205.3679556 +60.2576198 14.8792 0.0002 14.8792 0.0002 14.9100 0.0002 0.000389709 2.80088e-07 0.0003095117 1.974931e-07 -1.48 -0.72 5707 16.3802 4.474 0 -""" - - -def _load_script(name, path): - sys.path.insert(0, str(SCRIPTS)) - try: - spec = importlib.util.spec_from_file_location(f"_{name}", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - finally: - sys.path.remove(str(SCRIPTS)) - return module - - -merge_final_cat = _load_script("merge_final_cat", - SCRIPTS / "merge_final_cat.py") -create_final_cat = _load_script( - "create_final_cat", - REPO_ROOT / "scripts" / "python" / "create_final_cat.py") - - -@pytest.fixture(scope="module") -def param_list(): - return create_final_cat.read_param_file(str(CFIS / "final_cat.param")) - - -@pytest.fixture(scope="module") -def sextractor_columns(): - """Column names the tile SExtractor run writes (vector sizes stripped).""" - names = set() - for line in (CFIS / "default_noimaflags.param").read_text().splitlines(): - entry = line.split("#")[0].strip() - if entry: - names.add(re.sub(r"\(.*\)$", "", entry)) - return names - - -@pytest.fixture(scope="module") -def catalogue_columns(tmp_path_factory): - """Detection columns of a tile under tile_detection: unions_catalogue.""" - tmp = tmp_path_factory.mktemp("dr6") - cat, img, out = (tmp / "CFIS.202.301.r.cat", tmp / "CFIS.202.301.r.fits", - tmp / "sexcat-202-301.fits") - cat.write_text(DR6_CATALOGUE) - fits.PrimaryHDU(np.zeros((8, 8), dtype=np.float32)).writeto(img) - rs.make_ldac_from_ascii(str(cat), str(img), str(out), stamp_size=3) - with fits.open(out) as hdul: - return set(hdul["LDAC_OBJECTS"].columns.names) - {"VIGNET"} - - -def test_sextractor_only_is_exactly_what_the_catalogue_lacks( - param_list, sextractor_columns, catalogue_columns): - """Every requested SExtractor column the catalogue lacks, and no other.""" - lacking = {c for c in param_list - if c in sextractor_columns and c not in catalogue_columns} - assert set(merge_final_cat.SEXTRACTOR_ONLY_COLUMNS) == lacking - - -def test_catalogue_mode_request_is_carried( - param_list, sextractor_columns, catalogue_columns): - """Each requested detection column is one the converter writes.""" - requested = merge_final_cat.requested_columns(param_list, - "unions_catalogue") - detection = [c for c in requested if c in sextractor_columns] - assert detection and set(detection) <= catalogue_columns - # Only detection columns are dropped, and the order is the param file's. - assert requested == [c for c in param_list - if c not in merge_final_cat.SEXTRACTOR_ONLY_COLUMNS] - assert {"MAG_AUTO", "MAGERR_AUTO", "FLUX_RADIUS"} <= set(requested) - - -def test_sextractor_mode_requests_the_param_file(): - for input_type in ("cfis", "cfis_image_sims"): - params = create_final_cat.read_param_file( - str(REPO_ROOT / "workflow" / "config" / input_type - / "final_cat.param")) - assert merge_final_cat.requested_columns( - params, "sextractor") == params - - -def test_unknown_mode_is_refused(param_list): - with pytest.raises(ValueError): - merge_final_cat.requested_columns(param_list, "sextractr") diff --git a/tests/module/test_read_ext_sexcat.py b/tests/module/test_read_ext_sexcat.py deleted file mode 100644 index fa406b9a6..000000000 --- a/tests/module/test_read_ext_sexcat.py +++ /dev/null @@ -1,424 +0,0 @@ -"""UNIT TESTS FOR MODULE PACKAGE: READ_EXT_SEXCAT. - -Drives ``make_ldac_from_ascii`` on a synthetic ASCII SExtractor-format -catalogue and a synthetic tile image, and checks the FITS-LDAC it writes is -what the tile chain downstream of ``tile_detect`` reads: the LDAC_IMHEAD -extension carrying the tile header, the SExtractor column aliases, one -``VIGNET`` stamp per object cut from the image, and the input ``NUMBER`` -kept as is. It follows the catalogue through -``make_cat.save_sextractor_data``, which builds ``TILE_UNIQUE_ID``. The rest -covers the segmentation map: relabelling it to the catalogue's ``NUMBER`` and -setting neighbours' ``VIGNET`` pixels to -1e30, as SExtractor does, which is -all ngmix reads to mask neighbours. -""" - -from pathlib import Path - -import numpy as np -import numpy.testing as npt -import pytest -from astropy.io import fits - -from shapepipe.modules.make_cat_package import make_cat -from shapepipe.modules.read_ext_sexcat_package import read_ext_sexcat as rs - -NX, NY = 40, 30 -STAMP = 5 -# (NUMBER, X_IMAGE, Y_IMAGE): an interior object, one on the left edge, one -# in the top-right corner. -OBJECTS = [(1, 10.0, 12.0), (2, 1.0, 20.0), (7, 40.0, 30.0)] - - -def _write_ascii_cat(path): - lines = [ - "# 1 NUMBER Running object number", - "# 2 X_IMAGE Object position along x [pixel]", - "# 3 Y_IMAGE Object position along y [pixel]", - "# 4 ALPHA_J2000 Right ascension of barycenter [deg]", - "# 5 DELTA_J2000 Declination of barycenter [deg]", - "# 6 MAG_AUTO Kron-like elliptical aperture magnitude [mag]", - ] - for num, x, y in OBJECTS: - lines.append(f"{num} {x} {y} {150.0 + num} {30.0 + num} {20.0 + num}") - path.write_text("\n".join(lines) + "\n") - - -def _write_image(path): - # pixel value = 1000*row + column (0-based), so every stamp pixel names - # where it came from. - data = (np.arange(NY)[:, None] * 1000 + np.arange(NX)[None, :]).astype( - np.float32 - ) - hdu = fits.PrimaryHDU(data) - hdu.header["HISTORY"] = "input image 2605805p.fits" - hdu.header["TILEKEY"] = "kept" - hdu.writeto(path, overwrite=True) - - -@pytest.fixture -def ldac(tmp_path): - cat = tmp_path / "CFIS_cat-301-279.cat" - img = tmp_path / "CFIS_image-301-279.fits" - out = tmp_path / "sexcat-301-279.fits" - _write_ascii_cat(cat) - _write_image(img) - rs.make_ldac_from_ascii( - str(cat), str(img), str(out), stamp_size=STAMP - ) - return out - - -def test_ldac_layout_and_header(ldac): - with fits.open(ldac) as hdul: - assert [h.name for h in hdul] == ["PRIMARY", "LDAC_IMHEAD", "LDAC_OBJECTS"] - cards = hdul["LDAC_IMHEAD"].data[0][0] - assert isinstance(cards, str) or cards.ndim == 1 - text = "".join(cards) if not isinstance(cards, str) else cards - assert "TILEKEY" in text and "2605805p" in text - - -def test_number_is_kept_and_aliases_added(ldac): - with fits.open(ldac) as hdul: - data = hdul["LDAC_OBJECTS"].data - # The input NUMBER (gapped here: 1, 2, 7) is the object's identity and is - # copied unchanged; the converter builds no ID of its own. - npt.assert_array_equal(data["NUMBER"], [o[0] for o in OBJECTS]) - assert "TILE_UNIQUE_ID" not in data.names - npt.assert_array_equal(data["XWIN_IMAGE"], data["X_IMAGE"]) - npt.assert_array_equal(data["YWIN_IMAGE"], data["Y_IMAGE"]) - npt.assert_array_equal(data["XWIN_WORLD"], data["ALPHA_J2000"]) - npt.assert_array_equal(data["YWIN_WORLD"], data["DELTA_J2000"]) - - -def test_vignets_are_cut_from_the_image_and_padded_as_sextractor(ldac): - with fits.open(ldac) as hdul: - vignets = hdul["LDAC_OBJECTS"].data["VIGNET"] - assert vignets.shape == (len(OBJECTS), STAMP, STAMP) - - # Interior object at (10, 12), 1-based: centre pixel is (row 11, col 9). - assert vignets[0, STAMP // 2, STAMP // 2] == 11 * 1000 + 9 - assert vignets[0, 0, 0] == 9 * 1000 + 7 - - # Left edge, x = 1: the two columns left of the image are -1e30, the value - # SExtractor writes off the image. - assert (vignets[1, :, :2] == rs.BIG).all() - assert vignets[1, STAMP // 2, STAMP // 2] == 19 * 1000 + 0 - - # Top-right corner: only the lower-left quadrant of the stamp is in the - # image. - assert (vignets[2, STAMP // 2 + 1:, :] == rs.BIG).all() - assert (vignets[2, :, STAMP // 2 + 1:] == rs.BIG).all() - assert vignets[2, STAMP // 2, STAMP // 2] == 29 * 1000 + 39 - - -def test_tile_unique_id_reaches_the_final_catalogue(ldac, tmp_path): - """make_cat builds the ID from the tile and the catalogue's own NUMBER.""" - final = make_cat.prepare_final_cat_file(str(tmp_path), "-301-279") - n_obj = make_cat.save_sextractor_data(final, str(ldac)) - assert n_obj == len(OBJECTS) - with fits.open(tmp_path / "final_cat-301-279.fits") as hdul: - data = hdul["RESULTS"].data - assert "VIGNET" not in data.names - npt.assert_array_equal( - data["TILE_UNIQUE_ID"], 301279 * 10**6 + np.array([1, 2, 7]) - ) - npt.assert_allclose(data["TILE_ID"], 301.279) - - -# --- the segmentation map ------------------------------------------------- - - -def _seg_map(): - """Two footprints, labelled 7 and 9, on a 20x20 sky.""" - seg = np.zeros((20, 20), dtype=np.int32) - seg[2:6, 2:6] = 7 - seg[12:18, 12:18] = 9 - return seg - - -@pytest.mark.decision("detection.catalogue_neighbour_marking") -class TestRelabel: - """The relabelled map carries each object's NUMBER on its own footprint. - - Segmentation labels are not the catalogue's NUMBER; the claim is by the - pixel under the object's position, which is all both maps share. - """ - - def test_each_object_owns_the_footprint_it_sits_in(self): - seg = _seg_map() - # Positions in an order that is NOT the label order, so a - # relabelling that merely renumbered would fail. - out, counts = rs.relabel_seg( - seg, number=np.array([1, 2]), - x_image=np.array([15.0, 4.0]), y_image=np.array([15.0, 4.0])) - assert counts["matched"] == 2 - assert set(np.unique(out[seg == 9])) == {1} - assert set(np.unique(out[seg == 7])) == {2} - assert np.all(out[seg == 0] == 0) - - def test_an_unclaimed_footprint_becomes_a_neighbour(self): - seg = _seg_map() - out, _ = rs.relabel_seg( - seg, number=np.array([1]), - x_image=np.array([4.0]), y_image=np.array([4.0])) - assert set(np.unique(out[seg == 9])) == {rs.NEIGHBOUR_LABEL} - assert set(np.unique(out[seg == 7])) == {1} - - def test_an_object_on_sky_gets_a_disc(self): - seg = _seg_map() - out, counts = rs.relabel_seg( - seg, number=np.array([1, 5]), - x_image=np.array([4.0, 10.0]), y_image=np.array([4.0, 10.0]), - fallback_radius=2) - assert counts["unclaimed"] == 1 - assert out[9, 9] == 5 - assert np.count_nonzero(out == 5) == np.count_nonzero( - np.add.outer(np.arange(-2, 3) ** 2, np.arange(-2, 3) ** 2) <= 4) - - def test_two_objects_in_one_footprint_both_keep_a_centre(self): - seg = _seg_map() - out, counts = rs.relabel_seg( - seg, number=np.array([1, 2]), - x_image=np.array([14.0, 16.0]), y_image=np.array([14.0, 16.0]), - fallback_radius=1) - assert counts == dict(matched=1, unclaimed=0, shared=1, off_image=0, - shared_pixel=0) - assert out[13, 13] == 1 and out[15, 15] == 2 - assert np.count_nonzero(out == 1) > np.count_nonzero(out == 2) - - def test_every_object_is_self_somewhere(self): - rng = np.random.default_rng(0) - seg = np.zeros((60, 60), dtype=np.int32) - for label in range(1, 12): - row, col = rng.integers(0, 55, size=2) - seg[row:row + 5, col:col + 5] = label - number = np.arange(1, 31) - x_image = rng.uniform(1, 60, size=30) - y_image = rng.uniform(1, 60, size=30) - out, counts = rs.relabel_seg(seg, number, x_image, y_image) - assert sum(counts[k] for k in - ("matched", "unclaimed", "shared", "off_image")) == 30 - for num in number: - assert np.any(out == num), f"object {num} has no self pixels" - assert set(np.unique(out)) <= set(number) | {0, rs.NEIGHBOUR_LABEL} - - def test_two_objects_on_one_pixel(self): - seg = _seg_map() - out, counts = rs.relabel_seg( - seg, number=np.array([4, 6]), x_image=np.array([10.0, 10.2]), - y_image=np.array([10.0, 10.1]), fallback_radius=1) - assert counts["shared_pixel"] == 1 - assert out[9, 9] == 4 - assert np.any(out == 6) - - @pytest.mark.parametrize("x, y", [(0.4, 5.0), (5.0, 21.0)]) - def test_a_position_off_the_image_is_counted(self, x, y): - out, counts = rs.relabel_seg( - _seg_map(), number=np.array([1]), x_image=np.array([x]), - y_image=np.array([y])) - assert counts["off_image"] == 1 - assert not np.any(out == 1) - - -# Objects on _seg_map(): 1 in footprint 7, 2 on sky between the footprints; -# footprint 9 is claimed by nobody. -SEG_OBJECTS = [(1, 4.0, 4.0), (2, 10.0, 9.0)] -SEG_STAMP = 21 - - -def _marked(number, x, y, seg=None): - image = np.ones(_seg_map().shape, np.float32) - seg = _seg_map() if seg is None else seg - relabelled, _ = rs.relabel_seg(seg, number, x, y) - return rs._extract_vignets(image, x, y, SEG_STAMP, seg=relabelled, - number=number) - - -def _stamp_of(array, x, y, fill): - """The SEG_STAMP stamp of ``array`` centred on 1-based (x, y).""" - half = SEG_STAMP // 2 - padded = np.pad(array, half, constant_values=fill) - row, col = int(round(y)) - 1, int(round(x)) - 1 - return padded[row:row + SEG_STAMP, col:col + SEG_STAMP] - - -@pytest.mark.decision("detection.catalogue_neighbour_marking") -def test_neighbour_footprints_become_big_and_nothing_else(): - number = np.array([o[0] for o in SEG_OBJECTS]) - x = np.array([o[1] for o in SEG_OBJECTS]) - y = np.array([o[2] for o in SEG_OBJECTS]) - vignets = _marked(number, x, y) - seg = _seg_map() - - # Object 1 in footprint 7: footprint 9 (unclaimed) is a neighbour; its - # own footprint and the sky keep their image values. - s = _stamp_of(seg, x[0], y[0], fill=-99) - v = vignets[0] - assert (v[s == 9] == rs.BIG).all() - assert (v[s == 7] == 1).all() - assert (v[s == -99] == rs.BIG).all() # off the image - # Object 2's disc is a catalogue object's pixels, so a neighbour too; the - # rest of the sky is untouched. - relabelled, _ = rs.relabel_seg(seg, number, x, y) - disc = _stamp_of(relabelled, x[0], y[0], fill=-99) == 2 - assert disc.any() and (v[disc] == rs.BIG).all() - assert (v[(s == 0) & ~disc] == 1).all() - - # Object 2 on sky: every footprint in its stamp is a neighbour; the sky, - # its own centre included, keeps its image values. - s = _stamp_of(seg, x[1], y[1], fill=-99) - v = vignets[1] - assert (v[(s == 7) | (s == 9)] == rs.BIG).all() - assert (v[s == 0] == 1).all() - assert v[SEG_STAMP // 2, SEG_STAMP // 2] == 1 - - -@pytest.mark.decision("detection.catalogue_neighbour_marking") -def test_a_claimed_neighbour_is_masked_by_its_number(): - """With both footprints claimed, each object masks the other's.""" - number, x, y = np.array([3, 8]), np.array([4.0, 15.0]), np.array([4.0, 15.0]) - vignets = _marked(number, x, y) - seg = _seg_map() - for i, (own, other) in enumerate([(7, 9), (9, 7)]): - s = _stamp_of(seg, x[i], y[i], fill=-99) - assert (vignets[i][s == other] == rs.BIG).all() - assert (vignets[i][s == own] == 1).all() - - -@pytest.mark.decision("detection.catalogue_neighbour_marking") -def test_converter_relabels_and_marks_from_compressed_map(tmp_path): - """End to end from a compressed map, as fetched from vos.""" - cat = tmp_path / "CFIS_cat-301-279.cat" - img = tmp_path / "CFIS_image-301-279.fits" - seg_in = tmp_path / "CFIS_seg-301-279.fitsfz" - out = tmp_path / "sexcat-301-279.fits" - lines = ["# 1 NUMBER", "# 2 X_IMAGE", "# 3 Y_IMAGE", - "# 4 ALPHA_J2000", "# 5 DELTA_J2000"] - lines += [f"{n} {x} {y} 150.0 30.0" for n, x, y in SEG_OBJECTS] - cat.write_text("\n".join(lines) + "\n") - fits.PrimaryHDU(np.ones((20, 20), np.float32)).writeto(img) - fits.HDUList([fits.PrimaryHDU(), - fits.CompImageHDU(_seg_map())]).writeto(seg_in) - - rs.make_ldac_from_ascii(str(cat), str(img), str(out), stamp_size=SEG_STAMP, - seg_path=str(seg_in)) - - seg = _seg_map() - number, x, y = (np.array(c) for c in zip(*SEG_OBJECTS)) - relabelled, _ = rs.relabel_seg(seg, number, x, y) - assert set(np.unique(relabelled[seg == 7])) == {1} - assert set(np.unique(relabelled[seg == 9])) == {rs.NEIGHBOUR_LABEL} - with fits.open(out) as hdul: - v = hdul["LDAC_OBJECTS"].data["VIGNET"][0] - s = _stamp_of(seg, 4.0, 4.0, fill=-99) - assert (v[s == 9] == rs.BIG).all() and (v[s == 7] == 1).all() - - fits.HDUList([fits.PrimaryHDU(), - fits.CompImageHDU(_seg_map()[:10])]).writeto( - seg_in, overwrite=True) - with pytest.raises(ValueError, match="one grid"): - rs.make_ldac_from_ascii(str(cat), str(img), str(out), - stamp_size=SEG_STAMP, seg_path=str(seg_in)) - - -DR6_PATCH = Path(__file__).parent / "data" / "dr6_202.301_seg_patch.fits" - - -@pytest.mark.decision("detection.catalogue_neighbour_marking") -def test_dr6_marks_every_neighbour_pixel_and_no_own_pixel(): - """On a crowded 200x200 patch of the real 202.301 map and catalogue. - - The raw labels are not NUMBER, so this checks the whole chain on real - data: 1-based positions, the centre-pixel claim, and the stamp geometry. - Every stamp fully on the patch is checked against the RAW map: pixels of - footprints other than the one under the object are all -1e30, its own - footprint and the sky are untouched. (SExtractor's own VIGNET, on a - SExtractor seg map of an image-sim tile, marks 93% of neighbour-label - pixels and 0.14% of own pixels: check_sex_vignet.py.) - """ - with fits.open(DR6_PATCH) as hdul: - seg = hdul["SEG"].data - objects = hdul["OBJECTS"].data - number = np.array(objects["NUMBER"]) - x, y = np.array(objects["X_IMAGE"]), np.array(objects["Y_IMAGE"]) - relabelled, counts = rs.relabel_seg(seg, number, x, y) - assert counts["matched"] == len(number) - - stamp, half = 51, 25 - image = np.ones(seg.shape, np.float32) - vignets = rs._extract_vignets(image, x, y, stamp, seg=relabelled, - number=number) - col, row = np.rint(x).astype(int) - 1, np.rint(y).astype(int) - 1 - full = ((col >= half) & (col < seg.shape[1] - half) - & (row >= half) & (row < seg.shape[0] - half)) - assert full.sum() >= 10 - - marked = {"neighbour": [0, 0], "own": [0, 0], "sky": [0, 0]} - for i in np.flatnonzero(full): - s = seg[row[i] - half:row[i] + half + 1, col[i] - half:col[i] + half + 1] - big = vignets[i] == rs.BIG - own = s[half, half] - for kind, where in (("neighbour", (s != 0) & (s != own)), - ("own", s == own), ("sky", s == 0)): - marked[kind][0] += (big & where).sum() - marked[kind][1] += where.sum() - assert marked["neighbour"][1] > 1000 - assert marked["neighbour"][0] == marked["neighbour"][1] - assert marked["own"][0] == 0 - assert marked["sky"][0] == 0 - - -# --- the runner's output against the completeness table ------------------- - - -def test_runner_output_matches_tile_detect_completeness(tmp_path, monkeypatch): - """The runner writes exactly the files ``tile_detect`` expects. - - Runs the real runner, segmentation map on, into a run dir and checks it - with ``completeness.check_counts`` under ``SP_TILE_DETECTION=unions_catalogue``, - so the table and the converter's outputs cannot drift apart. - """ - import configparser - import importlib.util - import logging - - from shapepipe.modules.read_ext_sexcat_runner import read_ext_sexcat_runner - - scripts = Path(__file__).resolve().parents[2] / "workflow" / "scripts" - spec = importlib.util.spec_from_file_location( - "_completeness", scripts / "completeness.py") - completeness = importlib.util.module_from_spec(spec) - spec.loader.exec_module(completeness) - - cat = tmp_path / "CFIS_cat-301-279.cat" - img = tmp_path / "CFIS_image-301-279.fits" - seg_in = tmp_path / "CFIS_seg-301-279.fitsfz" - lines = ["# 1 NUMBER", "# 2 X_IMAGE", "# 3 Y_IMAGE", - "# 4 ALPHA_J2000", "# 5 DELTA_J2000"] - lines += [f"{n} {x} {y} 150.0 30.0" for n, x, y in SEG_OBJECTS] - cat.write_text("\n".join(lines) + "\n") - fits.PrimaryHDU(np.ones((20, 20), np.float32)).writeto(img) - fits.HDUList([fits.PrimaryHDU(), - fits.CompImageHDU(_seg_map())]).writeto(seg_in) - - run_dir = tmp_path / "run_sp_tile_Rx" - out_dir = run_dir / "read_ext_sexcat_runner" / "output" - out_dir.mkdir(parents=True) - config = configparser.ConfigParser() - config["READ_EXT_SEXCAT_RUNNER"] = { - "SEGMENTATION": "True", "MAKE_POST_PROCESS": "False", - "VIGNET_SIZE": str(SEG_STAMP), - } - read_ext_sexcat_runner( - [str(cat), str(img), str(seg_in)], {"output": str(out_dir)}, - "-301-279", config, "READ_EXT_SEXCAT_RUNNER", - logging.getLogger("test"), - ) - - monkeypatch.setenv("SP_TILE_DETECTION", "unions_catalogue") - table = completeness.COMPLETENESS["tile_detect"]["unions_catalogue"] - expect = table["read_ext_sexcat_runner"]["expect"] - written = sorted(p.name for p in out_dir.iterdir()) - assert len(written) == expect, written - ok, details = completeness.check_counts("tile_detect", run_dir) - assert ok, details diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini deleted file mode 100644 index 906ddef02..000000000 --- a/workflow/config/cfis/config_tile_Uc.ini +++ /dev/null @@ -1,97 +0,0 @@ -# ShapePipe configuration file for tile object selection from the UNIONS -# per-tile catalogue (tile_detection: unions_catalogue in the workflow run -# config). Writes the same run dir as config_tile_Sx.ini, so that the tile -# chain downstream reads one path whichever detection mode produced it. - - -## Default ShapePipe options -[DEFAULT] - -# verbose mode (optional), default: True, print messages on terminal -VERBOSE = True - -# Name of run (optional) default: shapepipe_run -RUN_NAME = run_sp_tile_Sx - -# Add date and time to RUN_NAME, optional, default: True -RUN_DATETIME = False - - -## ShapePipe execution options -[EXECUTION] - -# Module name, single string or comma-separated list of valid module runner names -MODULE = read_ext_sexcat_runner - -# Run mode, SMP or MPI -MODE = SMP - - -## ShapePipe file handling options -[FILE] - -# Log file master name, optional, default: shapepipe -LOG_NAME = log_sp - -# Runner log file name, optional, default: shapepipe_runs -RUN_LOG_NAME = log_run_sp - -# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the -# dashed tile ID (e.g. -210-282). -NUMBER_LIST = $SP_UNIT_NUM - -# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN -INPUT_DIR = $SP_RUN/output - -# Output directory -OUTPUT_DIR = $SP_RUN/output - - -## ShapePipe job handling options -[JOB] - -# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 - -# Timeout value (optional), default is None, i.e. no timeout limit applied -TIMEOUT = 96:00:00 - - -## Module options - -[READ_EXT_SEXCAT_RUNNER] - -INPUT_DIR = $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output - -FILE_PATTERN = CFIS_cat, CFIS_image, CFIS_seg, log_exp_headers - -FILE_EXT = .cat, .fits, .fitsfz, .sqlite - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# File name suffix for the output sextractor files (optional) -SUFFIX = sexcat - -# Side length of the square postage stamp (vignet) extracted from the tile -# image, in pixels (must be odd). Default: 51 -# @sc [decision:postage_stamp_size] -VIGNET_SIZE = 51 - -# The third input is the catalogue's r-band segmentation map: set neighbours' -# VIGNET pixels to -1e30, as SExtractor does, so ngmix masks them -# @sc [decision:detection.catalogue_neighbour_marking] -SEGMENTATION = True - -## Post-processing - -# Necessary for tiles, to enable multi-exposure processing -# @sc [decision:detection.epoch_membership_ccd_bounds] -MAKE_POST_PROCESS = True - -# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y -WORLD_POSITION = ALPHA_J2000,DELTA_J2000 - -# Accepted CCD pixel bounds xmin,xmax,ymin,ymax (0-based, strict) -# @sc [decision:detection.epoch_membership_ccd_bounds] -CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py index afa05373a..0d4594ec4 100644 --- a/workflow/scripts/merge_final_cat.py +++ b/workflow/scripts/merge_final_cat.py @@ -52,18 +52,6 @@ argument validator and then falls through to the ordinary walk, so it is not a way to add one tile by hand.) -WHICH COLUMNS DEPEND ON WHERE THE TILE'S DETECTIONS CAME FROM. The param file -lists the columns of a SExtractor-mode tile catalogue. Under ``tile_detection: -unions_catalogue`` the detection columns are instead the UNIONS catalogue's own -(read_ext_sexcat copies them as they are), and that catalogue lacks some of the -SExtractor quantities the param file names; ``SEXTRACTOR_ONLY_COLUMNS`` lists -them, and ``requested_columns`` subtracts exactly that list in that mode. The -merge stays strict on everything else: a column still requested and absent -from a tile stops the merge. tests/module/test_final_cat_columns.py derives the -catalogue-mode columns by running the converter on the UNIONS catalogue's -header and asserts the list is exactly the requested SExtractor columns the -catalogue does not carry, so it cannot silently go stale in either direction. - WHICH TILES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set is the CAMPAIGN's: every tile both declared in ``tile_list`` and present in the index, which is exactly the Snakefile's TILES_READY, rebuilt here from the same @@ -99,41 +87,12 @@ # Same directory; the rule invokes this file by path, so it is sys.path[0]. import build_index import hdf5_reconcile -from completeness import TILE_DETECTIONS # /scripts/python/create_final_cat.py, from /workflow/scripts/this. CFC_PATH = (Path(__file__).resolve().parents[2] / "scripts" / "python" / "create_final_cat.py") -# Requested SExtractor columns the UNIONS per-tile catalogue (DR6) does not -# carry, so a unions_catalogue-mode tile catalogue cannot have them. None has a -# DR6 column measuring the same quantity: DR6's MAG_APER is a magnitude through -# its own aperture, not FLUX_APER; MAG_AUTO, MAGERR_AUTO and FLUX_RADIUS are -# the same SExtractor measurements in both modes and stay requested. -SEXTRACTOR_ONLY_COLUMNS = ( - "MAG_WIN", "MAGERR_WIN", # Gaussian-windowed magnitude - "FLUX_AUTO", "FLUXERR_AUTO", # DR6 carries the AUTO magnitude only - "FLUX_APER", "FLUXERR_APER", # ShapePipe's aperture, in flux - "SNR_WIN", # Gaussian-windowed SNR - "FWHM_IMAGE", "FWHM_WORLD", # Gaussian-core FWHM -) - - -def requested_columns(param_list: list, tile_detection: str) -> list: - """The columns a tile catalogue of this detection mode must carry. - - The param file's list, in its order, less ``SEXTRACTOR_ONLY_COLUMNS`` when - the detections are the UNIONS catalogue's (see the module docstring). - """ - if tile_detection not in TILE_DETECTIONS: - raise ValueError(f"tile_detection {tile_detection!r} is not one of " - f"{TILE_DETECTIONS}") - if tile_detection == "sextractor": - return list(param_list) - return [c for c in param_list if c not in SEXTRACTOR_ONLY_COLUMNS] - - def spval_group(campaign: str) -> str: """The hdf5 group the campaign's per-tile datasets live under. @@ -192,9 +151,6 @@ def main() -> None: help="names the campaign's group in the output file") p.add_argument("--param-file", required=True, type=Path, help="the input type's final_cat.param — the column list") - p.add_argument("--tile-detection", required=True, choices=TILE_DETECTIONS, - help="where the tile detections came from (config " - "tile_detection); selects the columns requested") p.add_argument("--hdu", type=int, default=1) p.add_argument("--snapshot-json", type=Path, default=None, help="sp run's code snapshot (bin/sp's " @@ -202,9 +158,7 @@ def main() -> None: args = p.parse_args() cfc = load_create_final_cat() - param_list = requested_columns( - cfc.read_param_file(str(args.param_file), verbose=False), - args.tile_detection) + param_list = cfc.read_param_file(str(args.param_file), verbose=False) if not param_list: sys.exit(f"merge_final_cat: no columns read from {args.param_file}") # read_data/copy_data read their knobs out of this dict, exactly as From cca72e0df74653675534de69c2ff09d8d2aa524b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:01:40 +0200 Subject: [PATCH 30/44] sextractor_runner: join detections to an external catalogue MATCH_CATALOGUE (an ASCII SExtractor catalogue of the same image) pairs each SExtractor row with its mutual nearest neighbour within MATCH_RADIUS pixels; paired rows take its NUMBER, unpaired rows leave the catalogue, and a SEG_VIGNET column, when present, is relabelled to the new numbering. Fewer than MATCH_MIN_FRACTION of the rows pairing stops the run: that is what a catalogue measured on other pixels (a DR5 image against DR6) looks like. The join runs before the post-processing keys epochs on NUMBER. Co-Authored-By: Claude Opus 5.5 --- .../sextractor_package/match_catalogue.py | 198 ++++++++++++++++++ src/shapepipe/modules/sextractor_runner.py | 20 ++ tests/module/test_match_catalogue.py | 158 ++++++++++++++ 3 files changed, 376 insertions(+) create mode 100644 src/shapepipe/modules/sextractor_package/match_catalogue.py create mode 100644 tests/module/test_match_catalogue.py diff --git a/src/shapepipe/modules/sextractor_package/match_catalogue.py b/src/shapepipe/modules/sextractor_package/match_catalogue.py new file mode 100644 index 000000000..23f4c5474 --- /dev/null +++ b/src/shapepipe/modules/sextractor_package/match_catalogue.py @@ -0,0 +1,198 @@ +"""MATCH CATALOGUE. + +Join a SExtractor tile catalogue to an external catalogue of the same image, +so the tile's rows carry the external catalogue's object numbers. + +The UNIONS tile catalogue (Stephen Gwyn's MegaPipe SExtractor run on the DR6 +tile) defines the object list and the ``NUMBER`` that the photometry and +photo-z catalogues share. ShapePipe runs its own SExtractor on the same image +with the same detection configuration, which reproduces that catalogue object +for object, and takes membership and ``NUMBER`` from it here: every +measurement (windowed positions, ``VIGNET`` and its neighbour marking, the +SExtractor columns) stays SExtractor's own. + +:Author: Cail Daley + +""" + +import numpy as np +from astropy.io import ascii as asc +from astropy.io import fits +from scipy.spatial import cKDTree + +# The SEG_VIGNET label of a footprint whose SExtractor row has no partner in +# the external catalogue, and so leaves the catalogue. Negative, so it never +# collides with a NUMBER; UberSeg only asks "self or not self". +UNMATCHED_LABEL = -1 + + +def mutual_nearest(x_a, y_a, x_b, y_b, radius): + """One-to-one pairs of mutual nearest neighbours closer than ``radius``. + + Parameters + ---------- + x_a, y_a : array_like + Positions of the first set + x_b, y_b : array_like + Positions of the second set, in the same units + radius : float + Largest separation of a pair + + Returns + ------- + numpy.ndarray + Indices into the first set of the paired rows, ascending + numpy.ndarray + Index into the second set of each one's partner + numpy.ndarray + Separation of each pair + + """ + a = np.column_stack([x_a, y_a]).astype(np.float64) + b = np.column_stack([x_b, y_b]).astype(np.float64) + if len(a) == 0 or len(b) == 0: + empty = np.zeros(0, np.int64) + return empty, empty, np.zeros(0) + dist, nearest_b = cKDTree(b).query(a) + _, nearest_a = cKDTree(a).query(b) + idx_a = np.arange(len(a)) + paired = (dist < radius) & (nearest_a[nearest_b] == idx_a) + return idx_a[paired], nearest_b[paired], dist[paired] + + +def relabel(seg_vignets, old_number, new_number): + """Map segmentation labels from one numbering to another. + + Parameters + ---------- + seg_vignets : numpy.ndarray + Segmentation stamps labelled with the old numbering, 0 on sky + old_number : array_like of int + Every old label in use (the catalogue's ``NUMBER`` before the join) + new_number : array_like of int + New label of each ``old_number``; ``UNMATCHED_LABEL`` for one that + leaves the catalogue + + Returns + ------- + numpy.ndarray + The stamps in the new numbering, int32; labels outside + ``old_number`` become ``UNMATCHED_LABEL`` + + """ + old_number = np.asarray(old_number, np.int64) + table = np.full(int(max(old_number.max(initial=0), + seg_vignets.max(initial=0))) + 1, + UNMATCHED_LABEL, np.int32) + table[0] = 0 + table[old_number] = new_number + if seg_vignets.min(initial=0) < 0: + raise ValueError("Segmentation stamps hold negative labels.") + return table[seg_vignets] + + +def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, + w_log=None): + """Take membership and ``NUMBER`` from an external catalogue. + + Each SExtractor row is paired with its mutual nearest neighbour in the + external catalogue (``X_IMAGE``, ``Y_IMAGE``, same image grid) within + ``radius`` pixels. Paired rows take the external ``NUMBER``; unpaired + rows leave the catalogue. A ``SEG_VIGNET`` column, when present, is + relabelled to the new numbering, with the footprints of rows that left + marked ``UNMATCHED_LABEL``. The catalogue is rewritten in place; every + other HDU and column is kept. + + The join is exact when both catalogues come from the same pixels: on the + DR6 image of tile 186.307 every SExtractor row pairs, at a median + separation of 4e-5 pixel. Run on different pixels (a DR5 image against + the DR6 catalogue) only ~95% pair, so a pairing fraction below + ``min_fraction`` stops the run rather than silently losing objects. + + Parameters + ---------- + cat_path : str + Path to the SExtractor FITS-LDAC catalogue, rewritten in place + ext_cat_path : str + Path to the external ASCII catalogue in SExtractor format, with + ``NUMBER``, ``X_IMAGE`` and ``Y_IMAGE`` + radius : float, optional + Largest separation of a pair, in pixels; default 1 + min_fraction : float, optional + Smallest acceptable fraction of SExtractor rows with a partner; + default 0.99 + w_log : logging.Logger, optional + Pipeline logger + + Returns + ------- + dict + ``n_sextractor``, ``n_external``, ``n_paired``, ``n_dropped`` + (SExtractor rows without a partner) and ``n_external_only`` + (external objects without a SExtractor row) + + Raises + ------ + ValueError + If fewer than ``min_fraction`` of the SExtractor rows pair + + @sc [decision:detection.tile_detection] + """ + ext = asc.read(ext_cat_path, format="sextractor", + include_names=["NUMBER", "X_IMAGE", "Y_IMAGE"]) + with fits.open(cat_path) as hdul: + hdus = [hdu.copy() for hdu in hdul] + objects = next(h for h in hdus if h.name == "LDAC_OBJECTS") + data = objects.data + + i_sex, i_ext, dist = mutual_nearest( + data["X_IMAGE"], data["Y_IMAGE"], ext["X_IMAGE"], ext["Y_IMAGE"], + radius, + ) + counts = dict( + n_sextractor=len(data), n_external=len(ext), n_paired=len(i_sex), + n_dropped=len(data) - len(i_sex), + n_external_only=len(ext) - len(i_sex), + ) + summary = ", ".join(f"{k}={v}" for k, v in counts.items()) + fraction = len(i_sex) / max(len(data), 1) + if fraction < min_fraction: + raise ValueError( + f"Only {fraction:.4f} of the SExtractor rows of {cat_path} pair" + + f" with {ext_cat_path} within {radius} px (minimum" + + f" {min_fraction}; {summary}). The catalogue must come from the" + + " same pixels as the image: check that the tile image is the" + + " release of the catalogue (DR6)." + ) + + old_number = np.asarray(data["NUMBER"]) + new_number = np.full(len(data), UNMATCHED_LABEL, np.int64) + new_number[i_sex] = np.asarray(ext["NUMBER"])[i_ext] + + columns = [] + for col in objects.columns: + array = data[col.name][i_sex] + if col.name == "NUMBER": + array = new_number[i_sex].astype(array.dtype) + elif col.name == "SEG_VIGNET": + array = relabel(np.asarray(data[col.name]), old_number, + new_number)[i_sex] + columns.append(fits.Column( + name=col.name, format=col.format, unit=col.unit, dim=col.dim, + array=array, + )) + new = fits.BinTableHDU.from_columns( + columns, header=objects.header, name="LDAC_OBJECTS", + ) + fits.HDUList( + [new if h is objects else h for h in hdus] + ).writeto(cat_path, overwrite=True) + + if w_log: + sep = np.quantile(dist, [0.5, 0.99]) if len(dist) else [np.nan] * 2 + w_log.info( + f"Matched {cat_path} to {ext_cat_path} within {radius} px:" + + f" {summary}; pair separation p50 {sep[0]:.2e} px," + + f" p99 {sep[1]:.2e} px" + ) + return counts diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index e45885b06..92599be4a 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -9,6 +9,7 @@ import re from shapepipe.modules.module_decorator import module_runner +from shapepipe.modules.sextractor_package import match_catalogue as mc from shapepipe.modules.sextractor_package import sextractor_script as ss from shapepipe.pipeline.execute import execute @@ -109,6 +110,25 @@ def sextractor_runner( # Parse SExtractor errors stdout, stderr = ss_inst.parse_errors(stderr, stdout) + # MATCH_CATALOGUE (optional, environment-expanded path; empty for none): + # take membership and NUMBER from that external catalogue of the same + # image, before the post-processing keys the epoch HDUs on NUMBER. + match_path = ( + config.getexpanded(module_config_sec, "MATCH_CATALOGUE") + if config.has_option(module_config_sec, "MATCH_CATALOGUE") + else "" + ) + if match_path: + mc.match_catalogue( + ss_inst.path_output_file, + match_path, + radius=config.getfloat(module_config_sec, "MATCH_RADIUS"), + min_fraction=config.getfloat( + module_config_sec, "MATCH_MIN_FRACTION" + ), + w_log=w_log, + ) + # Run sextractor post processing if config.getboolean(module_config_sec, "MAKE_POST_PROCESS"): pos_params = config.getlist(module_config_sec, "WORLD_POSITION") diff --git a/tests/module/test_match_catalogue.py b/tests/module/test_match_catalogue.py new file mode 100644 index 000000000..84105ea16 --- /dev/null +++ b/tests/module/test_match_catalogue.py @@ -0,0 +1,158 @@ +"""Joining the tile SExtractor catalogue to the UNIONS catalogue.""" + +import numpy as np +import pytest +from astropy.io import fits + +from shapepipe.modules.sextractor_package import match_catalogue as mc +from shapepipe.pipeline.config import CustomParser + +HEADER = """\ +# 1 NUMBER Running object number +# 2 X_IMAGE Object position along x [pixel] +# 3 Y_IMAGE Object position along y [pixel] +# 4 MAG_AUTO Kron-like elliptical aperture magnitude [mag] +""" + + +def _external(path, number, x, y): + rows = "".join(f"{n:10d} {xi:11.4f} {yi:11.4f} 24.0000\n" + for n, xi, yi in zip(number, x, y)) + path.write_text(HEADER + rows) + return str(path) + + +def _sexcat(path, x, y, seg_vignets=None): + """A FITS-LDAC sexcat: NUMBER 1..n at (x, y), 3x3 stamps.""" + n = len(x) + number = np.arange(1, n + 1, dtype=np.int32) + cols = [ + fits.Column(name="NUMBER", format="J", array=number), + fits.Column(name="X_IMAGE", format="E", array=np.asarray(x)), + fits.Column(name="Y_IMAGE", format="E", array=np.asarray(y)), + fits.Column(name="XWIN_IMAGE", format="D", + array=np.asarray(x) + 0.01), + fits.Column(name="VIGNET", format="9E", dim="(3,3)", + array=np.arange(n * 9, dtype=np.float32).reshape(n, 9)), + ] + if seg_vignets is not None: + cols.append(fits.Column(name="SEG_VIGNET", format="9J", dim="(3,3)", + array=seg_vignets.reshape(n, 9))) + imhead = fits.BinTableHDU.from_columns( + [fits.Column(name="Field Header Card", format="80A", + array=np.array(["SIMPLE = T"]))], + name="LDAC_IMHEAD") + objects = fits.BinTableHDU.from_columns(cols, name="LDAC_OBJECTS") + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto(path) + return str(path) + + +def _objects(path): + with fits.open(path) as hdul: + assert [h.name for h in hdul] == ["PRIMARY", "LDAC_IMHEAD", + "LDAC_OBJECTS"] + return hdul["LDAC_OBJECTS"].data.copy() + + +X = np.array([10.0, 50.0, 90.0, 130.0, 170.0]) +Y = np.array([20.0, 60.0, 100.0, 140.0, 180.0]) + + +def test_rows_take_the_external_number_one_to_one(tmp_path): + """Every row pairs with its object, whatever the external order.""" + cat = _sexcat(tmp_path / "sexcat.fits", X, Y) + order = [3, 0, 4, 1, 2] + numbers = [1001, 1002, 1003, 1004, 1005] + # Shuffled, offset by 1e-4 px, plus one object SExtractor did not detect. + ext = _external(tmp_path / "ext.cat", numbers + [1006], + list(X[order] + 1e-4) + [300.0], + list(Y[order] - 1e-4) + [300.0]) + counts = mc.match_catalogue(cat, ext) + data = _objects(cat) + expected = np.empty(5, int) + expected[order] = numbers + assert list(data["NUMBER"]) == list(expected) + assert counts == dict(n_sextractor=5, n_external=6, n_paired=5, + n_dropped=0, n_external_only=1) + # Every measurement is SExtractor's, row for row. + np.testing.assert_array_equal(data["XWIN_IMAGE"], X + 0.01) + np.testing.assert_array_equal(data["VIGNET"][2].ravel(), + np.arange(18, 27)) + + +def test_unpaired_rows_leave_the_catalogue(tmp_path): + """A detection with no object within the radius is dropped.""" + cat = _sexcat(tmp_path / "sexcat.fits", X, Y) + ext = _external(tmp_path / "ext.cat", [7, 8, 9, 10], + X[[0, 1, 2, 4]], Y[[0, 1, 2, 4]] + 0.9) + counts = mc.match_catalogue(cat, ext, radius=1.0, min_fraction=0.5) + data = _objects(cat) + assert list(data["NUMBER"]) == [7, 8, 9, 10] + np.testing.assert_allclose(data["X_IMAGE"], X[[0, 1, 2, 4]]) + assert counts["n_dropped"] == 1 and counts["n_external_only"] == 0 + + +def test_pairs_are_mutual_nearest_neighbours(tmp_path): + """Two detections near one object: only the nearer one takes it.""" + x = np.array([10.0, 10.6, 90.0]) + y = np.array([10.0, 10.0, 90.0]) + cat = _sexcat(tmp_path / "sexcat.fits", x, y) + ext = _external(tmp_path / "ext.cat", [5, 6], [10.1, 90.0], [10.0, 90.0]) + mc.match_catalogue(cat, ext, min_fraction=0.5) + data = _objects(cat) + assert list(data["NUMBER"]) == [5, 6] + np.testing.assert_allclose(data["X_IMAGE"], [10.0, 90.0]) + + +def test_too_few_pairs_stop_the_run(tmp_path): + """Below MATCH_MIN_FRACTION (other pixels: the DR5 image) the run fails.""" + cat = _sexcat(tmp_path / "sexcat.fits", X, Y) + ext = _external(tmp_path / "ext.cat", [1, 2, 3, 4], X[:4], Y[:4]) + with pytest.raises(ValueError, match="DR6"): + mc.match_catalogue(cat, ext, min_fraction=0.99) + # Nothing was written. + assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4, 5] + mc.match_catalogue(cat, ext, min_fraction=0.8) + assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4] + + +def test_seg_vignet_is_relabelled(tmp_path): + """SEG_VIGNET labels follow NUMBER; a dropped row's footprint is -1.""" + x, y = X[:3], Y[:3] + seg = np.array([ + [[0, 1, 1], [0, 1, 2], [0, 0, 3]], + [[2, 2, 0], [1, 2, 0], [0, 0, 0]], + [[3, 3, 3], [0, 3, 3], [2, 0, 0]], + ], dtype=np.int32) + cat = _sexcat(tmp_path / "sexcat.fits", x, y, seg) + # Row 2 (NUMBER 2) has no partner; rows 1 and 3 become 40 and 70. + ext = _external(tmp_path / "ext.cat", [70, 40], [x[2], x[0]], + [y[2], y[0]]) + mc.match_catalogue(cat, ext, min_fraction=0.5) + data = _objects(cat) + assert list(data["NUMBER"]) == [40, 70] + lut = {0: 0, 1: 40, 2: mc.UNMATCHED_LABEL, 3: 70} + expected = np.vectorize(lut.get)(seg[[0, 2]]) + np.testing.assert_array_equal( + np.asarray(data["SEG_VIGNET"]).reshape(2, 3, 3), expected) + # Each object's own centre label is its new NUMBER, as UberSeg needs. + centre = np.asarray(data["SEG_VIGNET"]).reshape(2, 3, 3)[:, 1, 1] + assert list(centre) == [40, 70] + + +def test_relabel_rejects_unknown_negative_labels(): + with pytest.raises(ValueError): + mc.relabel(np.array([[-1, 1]]), [1], [5]) + + +def test_match_catalogue_key_expands_to_empty_when_unset(monkeypatch): + """config_tile_Sx.ini's ${SP_MATCH_CATALOGUE:-} means "no join" unset.""" + parser = CustomParser() + parser.read_string("[S]\nMATCH = ${SP_MATCH_CATALOGUE:-}\n" + "BARE = $SP_MATCH_CATALOGUE\n") + monkeypatch.delenv("SP_MATCH_CATALOGUE", raising=False) + assert parser.getexpanded("S", "MATCH") == "" + with pytest.raises(ValueError): + parser.getexpanded("S", "BARE") + monkeypatch.setenv("SP_MATCH_CATALOGUE", "/a/CFIS_cat-186-307.cat") + assert parser.getexpanded("S", "MATCH") == "/a/CFIS_cat-186-307.cat" From 8274d9977b1531d47055cbe0040b47712e6fa2ce Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 04:01:42 +0200 Subject: [PATCH 31/44] Detect tiles with SExtractor on data too; drop the catalogue converter Data and image simulations both run tile_detect as SExtractor (config_tile_Sx.ini), so XWIN, FLAGS_WIN, the VIGNET neighbour markers and every SExtractor column are measured one way. Under tile_detection: unions_catalogue, tile_get_catalogue fetches only the UNIONS .cat and tile_detect joins its detections to it (SP_MATCH_CATALOGUE), keeping the shared NUMBER and TILE_UNIQUE_ID. On DR6 186.307 all 41,201 detections pair (median separation 4e-5 px); 45 catalogue objects, faint deblended children, have no detection. Gone: read_ext_sexcat and its tests, config_tile_Uc.ini, the runner link, the segmentation-map fetch, merge_final_cat's SEXTRACTOR_ONLY_COLUMNS and --tile-detection, and the detection.catalogue_neighbour_marking decision. completeness.py changes, so this lands at a campaign boundary. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 146 ++++++++---------- tests/unit/test_final_cat_merge_invariants.py | 2 +- tests/unit/test_postage_stamp_size.py | 11 -- tests/unit/test_workflow_tile_detection.py | 91 ++++------- tests/workflow/params_pin.json | 6 +- tests/workflow/test_dag.py | 18 +++ universes/committed.yaml | 1 - workflow/README.md | 16 +- workflow/Snakefile | 6 +- workflow/config.yaml | 7 +- workflow/config/cfis/config_tile_Gic.ini | 11 +- workflow/config/cfis/config_tile_Sx.ini | 22 +++ workflow/config/cfis/final_cat.param | 10 +- workflow/rules/tile.smk | 131 +++++++--------- workflow/scripts/completeness.py | 29 +--- 15 files changed, 229 insertions(+), 278 deletions(-) diff --git a/astra.yaml b/astra.yaml index e566a1253..2c58269a0 100644 --- a/astra.yaml +++ b/astra.yaml @@ -124,8 +124,7 @@ decisions: label: Postage-stamp size shared by vignets, ngmix stamps and PSF models rationale: >- Every stamp width is 51 px (about 9.5 arcsec): the tile VIGNET (cut - from the tile image by read_ext_sexcat in committed data, by SExtractor - under tile_detection = sextractor), the multi-epoch vignetmaker stamps + by the tile SExtractor run), the multi-epoch vignetmaker stamps ngmix fits, the exposure SExtractor VIGNET PSFEx trains on, and the PSFEx model stamp. Only the first two are coupled: ngmix lays the tile VIGNET's -1e30 neighbour markers over each epoch stamp pixel for pixel, @@ -134,7 +133,6 @@ decisions: Values: default_noimaflags.param#VIGNET = 51; default.param#VIGNET = 51; - READ_EXT_SEXCAT_RUNNER.VIGNET_SIZE = 51; VIGNETMAKER_RUNNER_RUN_1.STAMP_SIZE = 51; VIGNETMAKER_RUNNER_RUN_2.STAMP_SIZE = 51; PSF_SIZE = 51. @@ -157,10 +155,9 @@ decisions: FSCALE before the joint fit, which puts them on zero-point 30 only if FSCALE = 10^(-0.4 (PHOTZP - 30)). Nothing in the repo checks this, but it held to 0.02% on a sampled exposure (2114045p). ngmix magnitudes use - MAG_ZP = 30. In committed data the catalogue's tile MAG_AUTO is the - UNIONS catalogue's own; the fixed tile zero-point of 30 - (ZP_FROM_HEADER=False) applies only under tile_detection = sextractor, - and assumes the MegaPipe stacks are calibrated to 30. + MAG_ZP = 30. The tile SExtractor run uses a fixed zero-point of 30 + (ZP_FROM_HEADER=False), which assumes the MegaPipe stacks are + calibrated to 30. Values: MAG_ZEROPOINT = 30.0; config_tile_Sx.ini#SEXTRACTOR_RUNNER.ZP_FROM_HEADER = False; @@ -174,9 +171,9 @@ decisions: header_everywhere: label: Per-image header zero-points on tiles too description: >- - ZP_FROM_HEADER=True for tile SExtractor (tile_detection = - sextractor only); changes tile magnitudes only where a tile's header - zero-point differs from 30. ngmix MAG_ZP stays 30. + ZP_FROM_HEADER=True for tile SExtractor; changes tile magnitudes + only where a tile's header zero-point differs from 30. ngmix MAG_ZP + stays 30. prior_insights: des_psf_blacklist: @@ -247,9 +244,8 @@ analyses: shape_measurement.epoch_masked_fraction_cut). Tiles have no flag image, and ShapePipe's tile SExtractor run requests no IMAFLAGS_ISO (detection.detection_source_mode). Neighbour pixels are masked - separately, from the tile segmentation map - (detection.catalogue_neighbour_marking, - shape_measurement.blend_handling). Sky-fixed masks never touch + separately, from the -1e30 markers SExtractor writes into the tile + VIGNET (shape_measurement.blend_handling). Sky-fixed masks never touch pixels: a star halo changes no stamp. Values: config_exp_psfex.ini#SEXTRACTOR_RUNNER.FLAG_IMAGE = True; @@ -342,12 +338,11 @@ analyses: detection: description: >- Object detection on r-band tiles (the galaxy sample) and on - single-exposure CCDs (PSF-star candidates, always SExtractor). The tile - sample is Gwyn's UNIONS tile catalogue (committed for data) or - ShapePipe's own SExtractor run (image simulations), per tile_detection. - Every default_tile.sex / config_tile_Sx.ini setting below applies only - to the SExtractor run, which follows the MegaPipe parameters of that - catalogue; exposures keep ShapePipe's stock values. + single-exposure CCDs (PSF-star candidates), both by ShapePipe's own + SExtractor runs. The tile run follows the MegaPipe parameters of Gwyn's + UNIONS tile catalogue, and in committed data its detections are joined + to that catalogue (tile_detection); exposures keep ShapePipe's stock + values. inputs: - id: tile_stack type: data @@ -362,39 +357,49 @@ analyses: [tile_detection, detection_threshold_policy, deblending_policy, background_model, weight_map_usage, zero_weight_interpolation, detection_source_mode, - epoch_membership_ccd_bounds, catalogue_neighbour_marking, - photometry_parameters, + epoch_membership_ccd_bounds, photometry_parameters, spurious_detection_cleaning, blend_photometry_mask_type, saturation_level] decisions: tile_detection: - label: Source of the tile galaxy sample + label: Object list and NUMBER of the tile galaxy sample rationale: >- - Real data takes its tile sample from Gwyn's UNIONS per-tile - catalogue, converted to the sexcat the chain reads with its own - NUMBER kept, so shape, photometry and photo-z catalogues share one - object list and one ID (TILE_UNIQUE_ID). PSF stars still come from - ShapePipe's exposure-level SExtractor run, because external star - catalogues are too shallow. detection_source_mode and the tile - SExtractor settings of the other decisions here govern only the - sextractor option; epoch_membership_ccd_bounds applies to both. - Image simulations, which have no UNIONS catalogue, use sextractor. + Both options run ShapePipe's tile SExtractor, so data and image + simulations share one detection and measurement path: windowed + positions, VIGNET neighbour markers and every SExtractor column. + In real data each detection is then paired with its mutual nearest + neighbour within 1 px in Gwyn's UNIONS per-tile catalogue and takes + its NUMBER, so shape, photometry and photo-z catalogues share one + object list and one ID (TILE_UNIQUE_ID); unpaired detections leave + the catalogue. The catalogue is MegaPipe's SExtractor run on the + same DR6 image with the configuration of default_tile.sex + (detection_threshold_policy), so on tile 186.307 every one of the + 41,201 detections pairs, at a median separation of 4e-5 px. The 45 + catalogue objects with no detection (0.11%; 4 in 20 < MAG_AUTO < + 24.5) are faint deblended children inside bright parents, which + SExtractor 2.25 leaves merged; they have photometry but no shape. + The join needs the tile image to be the catalogue's release: on + the DR5 image of the same tile only ~95% of detections pair, and + below 99% the run stops. PSF stars come from the exposure-level + SExtractor run in both options. Image simulations, which have no + UNIONS catalogue, keep SExtractor's own NUMBER. Values: - input_types.data.tile_detection = unions_catalogue. + input_types.data.tile_detection = unions_catalogue; + config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_RADIUS = 1.0; + config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_MIN_FRACTION = 0.99. default: unions_catalogue options: unions_catalogue: - label: Gwyn's UNIONS per-tile catalogue, converted in place + label: Tile SExtractor joined to Gwyn's UNIONS per-tile catalogue sextractor: - label: SExtractor on the tile image + label: Tile SExtractor with its own NUMBER detection_threshold_policy: label: Detection significance, minimum area, matched filter rationale: >- - Tile SExtractor (tile_detection = sextractor, i.e. image - simulations) uses the MegaPipe tile-catalogue threshold, minimum + Tile SExtractor uses the MegaPipe tile-catalogue threshold, minimum area and filter, so that its galaxy sample reproduces the UNIONS - catalogue committed data adopts. Run on the DR6 image of tile - 186.307, it reproduces the DR6 catalogue to 99.89% completeness + catalogue whose NUMBER committed data adopts (tile_detection). Run + on the DR6 image of tile 186.307, it reproduces the DR6 catalogue to 99.89% completeness and 100% purity with identical segmentation pixels; the 45 misses are faint deblended children of bright parents (#929). This validates the configuration on data pixels, not detection on @@ -573,11 +578,9 @@ analyses: absent SExtractor falls back to its built-in level (50000 ADU per its documentation). Delivered exposures carry the card (every CCD of 2114045p: 65535) and split_exp keeps each CCD's header; a sampled - tile (CFIS.186.307) has 9558.7, which applies under tile_detection = - sextractor. The PSF-star masks require FLAGS == 0, so the level can - remove bright stars from the PSF sample. In committed data the final - catalogue's FLAGS column is the UNIONS catalogue's own, not this - pass's. Changing the card or pinning a fixed level requires checking + tile (CFIS.186.307) has 9558.7. The PSF-star masks require FLAGS == + 0, so the level can remove bright stars from the PSF sample. + Changing the card or pinning a fixed level requires checking the resulting bright-star selection. Values: default_exp.sex#SATUR_KEY = SATURATE; @@ -600,9 +603,7 @@ analyses: recorded. On the exposure pass MAG_AUTO is the axis of the star-selection magnitude windows and FLUX_AUTO is PSFEx's photometric normalisation, so a different Kron factor moves stars - across those cuts. The tile values apply under tile_detection = - sextractor; in committed data the catalogue's MAG_AUTO is the UNIONS - catalogue's own. The reported apertures and flux-fraction + across those cuts. The reported apertures and flux-fraction measurements must match the simulated catalogue they are compared with. Values: @@ -650,13 +651,10 @@ analyses: admits its upper endpoint, column 2080. [HARDCODED] The comparisons are strict in make_post_process; CCD_SIZE sets only the bounds. A WCS inversion failure skips that CCD, lowering N_EPOCH, which caps - how many exposures enter each galaxy's multi-epoch fit. Both - tile_detection options run this post-processing. + how many exposures enter each galaxy's multi-epoch fit. Values: SEXTRACTOR_RUNNER.CCD_SIZE = 33,2080,1,4612; - SEXTRACTOR_RUNNER.MAKE_POST_PROCESS = True; - READ_EXT_SEXCAT_RUNNER.CCD_SIZE = 33,2080,1,4612; - READ_EXT_SEXCAT_RUNNER.MAKE_POST_PROCESS = True. + SEXTRACTOR_RUNNER.MAKE_POST_PROCESS = True. default: trimmed_bounds_33_2080 options: trimmed_bounds_33_2080: @@ -664,36 +662,6 @@ analyses: inclusive_bounds: label: Same bounds, inclusive description: Not implemented; make_post_process uses strict comparisons. - catalogue_neighbour_marking: - label: Neighbour pixels in the catalogue path's VIGNET - rationale: >- - ngmix masks a neighbour only where the tile VIGNET is -1e30 (flag - 2**10: zero weight, noise-filled under noisefill), and SExtractor - writes -1e30 on neighbours' footprints and off the image. The - unions_catalogue converter reproduces that from the catalogue's own - r-band segmentation map (CFIS..r.seg.fits.fz, on the tile's - pixel grid), so both tile_detection options mask the same way. - Segmentation labels are not the catalogue NUMBER; each object claims - the footprint under its centre pixel (the VIGNET centre pixel), which - on 202.301 matches 36,064 of 36,065 objects with none shared. The map - is relabelled to NUMBER (unclaimed footprints -1; an object on sky or - on a claimed footprint gets a 3-pixel-radius disc of its own), and - VIGNET pixels whose relabelled value is neither 0 nor the object's - NUMBER become -1e30. Without it, catalogue-mode ngmix fits neighbour - light: in pilot smk-g10, NGMIX_MAG_NOSHEAR was over 1 mag brighter - than MAG_AUTO for 6.1% of objects (2.7% with SExtractor, g9). - Values: - READ_EXT_SEXCAT_RUNNER.SEGMENTATION = True. - default: segmentation_map - options: - segmentation_map: - label: -1e30 on other objects' segmentation footprints, as SExtractor - none: - label: Image pixels only; ngmix masks no neighbours - excluded: true - excluded_reason: >- - Leaves neighbour light in the fit, unlike the sextractor option - it replaces; the cause of the smk-g10 magnitude outliers. prior_insights: guinot22_sextractor_params: claim: >- @@ -815,8 +783,9 @@ analyses: object_position_columns: label: Windowed centroids (XWIN/YWIN) define every position rationale: >- - PSF interpolation sites, tile and multi-epoch stamp centres, and the - catalogue position all use SExtractor's windowed centroid: in pixels + PSF interpolation sites, tile and multi-epoch stamp centres, epoch + membership and the catalogue position all use the tile SExtractor + run's windowed centroid, in data and image simulations alike: in pixels for tile stamps and exposure-side PSF validation, in world coordinates for tile-side PSF interpolation and multi-epoch stamps. Windowed, isophotal and model centroids differ systematically for @@ -835,6 +804,15 @@ analyses: options: xwin_windowed: label: Windowed centroids everywhere + isophotal_barycentre: + label: Isophotal barycentre (X_IMAGE/Y_IMAGE) + excluded: true + excluded_reason: >- + Noisier than the windowed centroid and pulled by asymmetric or + deblend-clipped footprints; on the DR6 image of tile 186.307 + it sits 0.16 px (median) and 1.6 px (p99) from it for + 20 < MAG_AUTO < 24.5, and the image simulations measure the + windowed one. stamp_positioning_and_padding: label: Nearest-pixel stamp extraction with zero padding rationale: >- @@ -2047,7 +2025,7 @@ analyses: excluded_reason: >- Not needed in the pipeline: the cut needs only each object's TILE_ID and position, and sp_validation applies it downstream. - Leaving it out keeps every UNIONS catalogue row in the shape + Leaving it out keeps every tile-catalogue row in the shape catalogue. failure_sentinels: label: Objects without shape measurements kept, with sentinel values diff --git a/tests/unit/test_final_cat_merge_invariants.py b/tests/unit/test_final_cat_merge_invariants.py index 21eb6adad..7dcfc6183 100644 --- a/tests/unit/test_final_cat_merge_invariants.py +++ b/tests/unit/test_final_cat_merge_invariants.py @@ -130,7 +130,7 @@ def _campaign(root: Path, drop=None): argv = [sys.executable, str(SCRIPT), "--products-dir", str(products), "--tile-list", str(tile_list), "--index-db", str(index), "--output", str(output), "--campaign", CAMPAIGN, - "--param-file", str(param), "--tile-detection", "sextractor"] + "--param-file", str(param)] return argv, output, sources diff --git a/tests/unit/test_postage_stamp_size.py b/tests/unit/test_postage_stamp_size.py index 0bd31b568..ca7d4af72 100644 --- a/tests/unit/test_postage_stamp_size.py +++ b/tests/unit/test_postage_stamp_size.py @@ -11,12 +11,6 @@ CFIS_CONFIG = CONFIG / "cfis" -def _ini_int(path, section, option): - parser = configparser.ConfigParser(interpolation=None) - assert parser.read(path) == [str(path)] - return parser.getint(section, option) - - def _multi_epoch_stamp_size(path): """STAMP_SIZE of the one vignetmaker section run in MULTI-EPOCH mode.""" parser = configparser.ConfigParser(interpolation=None) @@ -50,11 +44,6 @@ def test_tile_and_epoch_vignet_sizes_match(): """The tile markers align pixel-for-pixel with each multi-epoch stamp.""" param_file = CFIS_CONFIG / "default_noimaflags.param" sizes = { - "config_tile_Uc.ini#READ_EXT_SEXCAT_RUNNER.VIGNET_SIZE": _ini_int( - CFIS_CONFIG / "config_tile_Uc.ini", - "READ_EXT_SEXCAT_RUNNER", - "VIGNET_SIZE", - ), f"{param_file.name}#VIGNET": _param_vignet_size(param_file), } diff --git a/tests/unit/test_workflow_tile_detection.py b/tests/unit/test_workflow_tile_detection.py index 24775304e..498a015de 100644 --- a/tests/unit/test_workflow_tile_detection.py +++ b/tests/unit/test_workflow_tile_detection.py @@ -1,13 +1,13 @@ -"""The two tile_detect modes agree on where the sexcat is. - -``tile_detection: unions_catalogue`` swaps the SExtractor rule for a fetch plus -a conversion, and everything downstream of ``tile_detect`` is unchanged only -because both modes write ``run_sp_tile_Sx`` and both are checked as the -``tile_detect`` stage. The agreements that make that true are between files -that never see each other at run time: the two inis' RUN_NAMEs, -``completeness.STAGE_DIR``, the flavoured ``COMPLETENESS['tile_detect']`` -table, and ``run_report``'s stage list. They are asserted here, statically. -Container-free: the scripts are stdlib-only. +"""The two tile_detection modes run one tile_detect, and agree on its inputs. + +Both modes run SExtractor (config_tile_Sx.ini) in the ``run_sp_tile_Sx`` stage +dir; ``tile_detection: unions_catalogue`` adds ``tile_get_catalogue`` and +points the ini's MATCH_CATALOGUE at its output through the workflow's +SP_MATCH_CATALOGUE. The agreements that make that work are between files that +never see each other at run time: the inis' RUN_NAMEs and fetch patterns, +``completeness``'s tables, ``run_report``'s stage list and tile.smk's path. +They are asserted here, statically. Container-free: the scripts are +stdlib-only. """ import configparser @@ -44,43 +44,35 @@ def _ini(name): def test_modes_are_the_two_the_rules_know(): assert completeness.TILE_DETECTIONS == ("sextractor", "unions_catalogue") - assert set(completeness.COMPLETENESS["tile_detect"]) == set( - completeness.TILE_DETECTIONS) -def test_both_detection_inis_write_the_stage_dir(): - """config_tile_Sx and config_tile_Uc share the run dir STAGE_DIR names.""" +def test_detection_ini_writes_the_stage_dir(): level, subdir = completeness.STAGE_DIR["tile_detect"] assert level == "tile" - for name in ("config_tile_Sx.ini", "config_tile_Uc.ini"): - assert _ini(name)["DEFAULT"]["RUN_NAME"].strip() == subdir, ( - f"{name} writes a run dir other than {subdir}: unit_pre would clear " - "the wrong directory and the chain downstream would read nothing.") - assert _ini("config_tile_Uc.ini")["DEFAULT"]["RUN_DATETIME"].strip() == "False" + assert _ini("config_tile_Sx.ini")["DEFAULT"]["RUN_NAME"].strip() == subdir -def test_fetch_ini_matches_its_stage_and_feeds_the_converter(): +def test_detection_joins_only_when_the_workflow_says_so(): + """MATCH_CATALOGUE is empty unless SP_MATCH_CATALOGUE is exported.""" + sx = _ini("config_tile_Sx.ini")["SEXTRACTOR_RUNNER"] + assert sx["MATCH_CATALOGUE"].strip() == "${SP_MATCH_CATALOGUE:-}" + assert float(sx["MATCH_RADIUS"]) == 1.0 + assert 0.9 < float(sx["MATCH_MIN_FRACTION"]) <= 1 + # The join renumbers before the post-processing keys epochs on NUMBER. + assert sx["MAKE_POST_PROCESS"].strip() == "True" + + +def test_fetch_ini_writes_the_catalogue_tile_smk_joins(): + """Gic's output is the path tile.smk exports as SP_MATCH_CATALOGUE.""" level, gic = completeness.STAGE_DIR["tile_get_catalogue"] assert level == "tile" assert _ini("config_tile_Gic.ini")["DEFAULT"]["RUN_NAME"].strip() == gic - uc = _ini("config_tile_Uc.ini")["READ_EXT_SEXCAT_RUNNER"] - assert f"$SP_RUN/output/{gic}/get_images_runner/output" in uc["INPUT_DIR"] - assert uc["FILE_PATTERN"].split(",")[0].strip() == "CFIS_cat" - # The segmentation map rides the same fetch and is the converter's third - # input, the position SEGMENTATION = True reads it from. gi = _ini("config_tile_Gic.ini")["GET_IMAGES_RUNNER"] - assert [p.strip() for p in gi["OUTPUT_FILE_PATTERN"].split(",")] == [ - "CFIS_cat-", "CFIS_seg-"] - assert [e.strip() for e in gi["INPUT_FILE_EXT"].split(",")] == [ - ".cat", ".fits.fz"] - third = [[v.strip() for v in uc[k].split(",")][2] - for k in ("INPUT_DIR", "FILE_PATTERN", "FILE_EXT")] - assert third == [f"$SP_RUN/output/{gic}/get_images_runner/output", - "CFIS_seg", ".fitsfz"] - assert uc["SEGMENTATION"].strip() == "True" - # The multi-epoch post-processing is what gives the sexcat its EPOCH_k - # extensions; ngmix_range.py refuses a sexcat without them. - assert uc["MAKE_POST_PROCESS"].strip() == "True" + assert gi["OUTPUT_FILE_PATTERN"].strip() == "CFIS_cat-" + assert gi["INPUT_FILE_EXT"].strip() == ".cat" + smk = (REPO_ROOT / "workflow" / "rules" / "tile.smk").read_text() + assert f'/output/{gic}/get_images_runner/output"' in smk + assert 'f"{gic}/CFIS_cat{unit_num(tile)}.cat"' in smk def _stage_dir(tmp_path, runner, n): @@ -91,31 +83,16 @@ def _stage_dir(tmp_path, runner, n): return tmp_path -@pytest.mark.parametrize("mode, runner, expect", [ - ("sextractor", "sextractor_runner", 2), - ("unions_catalogue", "read_ext_sexcat_runner", 1), +@pytest.mark.parametrize("stage, runner, expect", [ + ("tile_detect", "sextractor_runner", 2), + ("tile_get_catalogue", "get_images_runner", 1), ]) -def test_tile_detect_is_checked_per_mode(tmp_path, monkeypatch, mode, runner, expect): - monkeypatch.setenv("SP_TILE_DETECTION", mode) +def test_stage_counts(tmp_path, stage, runner, expect): ok, details = completeness.check_counts( - "tile_detect", _stage_dir(tmp_path, runner, expect)) + stage, _stage_dir(tmp_path, runner, expect)) assert ok and details == [(runner, expect, expect, False)] -def test_unset_mode_is_sextractor(tmp_path, monkeypatch): - """A prologue without the export checks as before: data runs are unchanged.""" - monkeypatch.delenv("SP_TILE_DETECTION", raising=False) - ok, details = completeness.check_counts( - "tile_detect", _stage_dir(tmp_path, "sextractor_runner", 2)) - assert ok and details[0][0] == "sextractor_runner" - - -def test_invalid_mode_is_fatal(tmp_path, monkeypatch): - monkeypatch.setenv("SP_TILE_DETECTION", "steven") - with pytest.raises(ValueError, match="SP_TILE_DETECTION"): - completeness.check_counts("tile_detect", tmp_path) - - @pytest.mark.parametrize("mode, present", [ ("sextractor", False), ("unions_catalogue", True), (None, False), ]) diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index eeb61d950..4502a6f20 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -8,10 +8,10 @@ "exp_persist": "302e2837542bc1102430c27c81c600b7cda32e8bddcb5fd60d33950987609fff", "exp_psf": "2c4f6d00f1939ccbf05b4982a0202a0ff92a4373a727f4e00f3aaab4eba03352", "exp_split": "6e954f8f3d06f44d3f164675912ce27d6216648855d04968f9168bd7d0f2c4fa", - "final_cat_merge": "f6222c98be9777cbab4131d8857bd3817719df9302d0a321c6cd99f67d5bdc76", + "final_cat_merge": "e7f46859c4503a2220713d7bb2507555515d0a9632d780b20f14c59e32210023", "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", - "tile_detect": "1b4b0871bf28296f8e5d6855ae47ff2cb84542c93593a5ae5c0ffc3c84c45479", + "tile_detect": "530a89d61ca61f42d660d2b1d9c1c9926d9b552618bb1a131922b2473f5aa9f2", "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", @@ -24,7 +24,7 @@ "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e", + "sha256": "ce48433c4ecce26c999d09705861eeb05625e09ad5bf3383213192729fc1f2c7", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index 46ae93fc0..69a70d17c 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -31,6 +31,24 @@ def test_rule_set_matches_input_mode(campaign, dag): assert "merge_final_cats" not in dag.declared_rule_names +def test_tile_detect_joins_the_catalogue_iff_unions(campaign, dag): + """Data's tile_detect waits on the fetch and exports its catalogue as + SP_MATCH_CATALOGUE; the image-simulation prologue exports nothing new.""" + for job in dag.jobs_for("tile_detect"): + tile = job.wildcards.tile + inputs = {str(f) for f in job.input} + fetch = str(campaign.tile_manifest(tile, "tile_get_catalogue")) + if campaign.tile_detection == "unions_catalogue": + gic = (campaign.run_dir / "tiles" / tile[:2] / tile / "output" + / "run_sp_tile_Gic" / "get_images_runner" / "output") + cat = gic / f"CFIS_cat-{tile.replace('.', '-')}.cat" + assert fetch in inputs + assert f"export SP_MATCH_CATALOGUE='{cat}'" in job.params.pre + else: + assert fetch not in inputs + assert "SP_MATCH_CATALOGUE" not in job.params.pre + + def test_clean_exposure_waits_on_persist_iff_psf(campaign, dag): """Reclamation waits for persistence and exactly its in-scope readers.""" jobs = dag.jobs_for("clean_exposure") diff --git a/universes/committed.yaml b/universes/committed.yaml index bc34d7601..05d88e097 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -24,7 +24,6 @@ analyses: photometry_parameters: kron_25_35 detection_source_mode: sx_nomask_single_image epoch_membership_ccd_bounds: trimmed_bounds_33_2080 - catalogue_neighbour_marking: segmentation_map preparation: decisions: astrometric_solution_source: delivered_headers diff --git a/workflow/README.md b/workflow/README.md index a1654fcb4..4f0b135c3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -28,13 +28,13 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' # `psf_model` is `psfex` for data (`fake` for image sims). `mccd` is refused # until `persist_exp.py` and `merge_star_cat.py` read MCCD products. PSFEx is # exercised by smk-g4 through smk-g6. -# `tile_detection` is `unions_catalogue` (the input_types default for data: -# the UNIONS per-tile catalogue at `inputs.catalogues` is fetched and -# converted in place, keeping its NUMBER, and its segmentation map sets -# neighbours' VIGNET pixels to -1e30 as SExtractor does) or `sextractor` (the -# tile is detected with SExtractor; the default for image sims). Either way -# make_cat writes TILE_UNIQUE_ID = tile_id * 10**6 + NUMBER and ngmix masks -# the same neighbours. +# Both `tile_detection` values run SExtractor on the tile image. Under +# `unions_catalogue` (the input_types default for data) the UNIONS per-tile +# catalogue at `inputs.catalogues` is fetched and the detections are joined +# to it, taking its NUMBER; the tile image must be the catalogue's release +# (DR6), or the join fails. `sextractor` (the default for image sims) keeps +# SExtractor's own NUMBER. Either way make_cat writes +# TILE_UNIQUE_ID = tile_id * 10**6 + NUMBER. # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -240,7 +240,7 @@ workflow/ rules/ prepare.smk tile get_images/uncompress/find_exposures exposure.smk per-exposure: get_images, split, psf, persist (no temp()); campaign star_cat_merge - tile.smk per-tile: exp forest, merge_headers, detect (SExtractor, or fetch + convert the UNIONS catalogue), vignets, ngmix, merge, make_cat; campaign final_cat_merge + tile.smk per-tile: exp forest, merge_headers, detect (SExtractor, joined to the UNIONS catalogue on data), vignets, ngmix, merge, make_cat; campaign final_cat_merge scripts/ build_index.py prepare-phase run_index.sqlite builder (plain script) build_forest.py per-tile exposure symlink forest (group-compatible shell) diff --git a/workflow/Snakefile b/workflow/Snakefile index 6aed81cff..ac9d0760c 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -175,9 +175,9 @@ sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 from completeness import STAGE_DIR, TILE_DETECTIONS # noqa: E402 -# Where the tile's galaxy sample comes from: SExtractor on the tile image, or -# the UNIONS per-tile catalogue fetched and converted in place (tile.smk). -# Both write run_sp_tile_Sx, so nothing downstream of tile_detect branches. +# How the tile's galaxy sample is defined: SExtractor on the tile image, alone +# or joined to the UNIONS per-tile catalogue (tile.smk). Both write +# run_sp_tile_Sx, so nothing downstream of tile_detect branches. TILE_DETECTION = config.get("tile_detection", "sextractor") if TILE_DETECTION not in TILE_DETECTIONS: raise WorkflowError( diff --git a/workflow/config.yaml b/workflow/config.yaml index b6f22a01e..8f2cd90c1 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -21,9 +21,10 @@ input_type: data # run: smk-g6 # Default settings per input_type, can be overridden by a user-defined run config file. -# - tile_detection: allowed are `unions_catalogue` (fetch official UNIONS catalogue -# from vos; recommended for data); `sextractor` (detect objects via a SExtractor -# module run; required for image_sims) +# - tile_detection: both run SExtractor on the tile. `unions_catalogue` (data) +# also fetches the UNIONS per-tile catalogue and joins the detections to it, +# taking its NUMBER (needs the DR6 tile image); `sextractor` keeps +# SExtractor's own (required for image_sims) # - psf_model: allowed are `psfex` (default), `mccd`; `fake` (image_sims only, # PSF read from psf_dict) input_types: diff --git a/workflow/config/cfis/config_tile_Gic.ini b/workflow/config/cfis/config_tile_Gic.ini index 1e6bdb7f5..a9cb21654 100644 --- a/workflow/config/cfis/config_tile_Gic.ini +++ b/workflow/config/cfis/config_tile_Gic.ini @@ -1,5 +1,4 @@ # ShapePipe configuration file for: get the UNIONS per-tile object catalogue -# and its r-band segmentation map # (tile_detection: unions_catalogue in the workflow run config) @@ -54,7 +53,7 @@ TIMEOUT = 96:00:00 ## Module options -# Get the external tile catalogue and its segmentation map +# Get the UNIONS tile catalogue (CFIS..r.cat) [GET_IMAGES_RUNNER] FILE_PATTERN = tile_numbers @@ -68,19 +67,19 @@ NUMBERING_SCHEME = # Where the catalogues are: the run config's inputs.catalogues, a local # directory or a vos: URL (vos:cfis/tiles_DR6) -INPUT_PATH = $SP_INPUT_CATALOGUES, $SP_INPUT_CATALOGUES +INPUT_PATH = $SP_INPUT_CATALOGUES # Input file pattern including tile number as dummy template -INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.seg +INPUT_FILE_PATTERN = CFIS.000.000.r # Input file extensions -INPUT_FILE_EXT = .cat, .fits.fz +INPUT_FILE_EXT = .cat # Input numbering scheme, python regexp INPUT_NUMBERING = \d{3}\.\d{3} # Output file pattern without number -OUTPUT_FILE_PATTERN = CFIS_cat-, CFIS_seg- +OUTPUT_FILE_PATTERN = CFIS_cat- # Copy/download method, one in 'vos', 'symlink'; the workflow derives it from # inputs.catalogues (vos: URL -> vos, local directory -> symlink) diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index aa5db396f..b73b07152 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -115,6 +115,28 @@ CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat +## Join to an external catalogue (optional) + +# Path to an ASCII SExtractor catalogue of the same image (NUMBER, X_IMAGE, +# Y_IMAGE): each SExtractor row pairs with its mutual nearest neighbour in it +# and takes its NUMBER; rows without a partner leave the catalogue. Empty for +# no join. The workflow sets SP_MATCH_CATALOGUE to the tile's UNIONS +# catalogue under tile_detection: unions_catalogue and leaves it unset +# otherwise (image simulations, tile_detection: sextractor). +# @sc [decision:detection.tile_detection] +MATCH_CATALOGUE = ${SP_MATCH_CATALOGUE:-} + +# Largest separation of a pair, in pixels. The same detection run on the same +# pixels pairs at ~1e-4 px, so the radius only has to exclude neighbours. +# @sc [decision:detection.tile_detection] +MATCH_RADIUS = 1.0 + +# Smallest fraction of SExtractor rows that must pair; fewer stops the run. +# A catalogue measured on other pixels (the DR5 image of a tile against its +# DR6 catalogue) pairs ~95%. +# @sc [decision:detection.tile_detection] +MATCH_MIN_FRACTION = 0.99 + ## Post-processing # Necessary for tiles, to enable multi-exposure processing diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 88a831b95..fb8b3f3f7 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -8,8 +8,9 @@ YWIN_WORLD # @sc [decision:catalogue_assembly.tile_overlap_handling] TILE_ID -# Object number within the tile: sp_validation's extraction joins on it, and -# the per-tile catalogue always carries it. +# Object number within the tile, the UNIONS catalogue's under tile_detection: +# unions_catalogue: sp_validation's extraction joins on it, and the per-tile +# catalogue always carries it. NUMBER # Survey-wide object ID, tile_id * 10**6 + NUMBER (make_cat, both @@ -226,10 +227,7 @@ NGMIX_FLUX_ERR_2P NGMIX_FLUX_ERR_NOSHEAR # magnitudes -# SExtractor detection columns. Under tile_detection: unions_catalogue the -# UNIONS catalogue supplies them, and the merge drops the ones it lacks -# (merge_final_cat.SEXTRACTOR_ONLY_COLUMNS: the WIN, AUTO-flux, APER and FWHM -# columns and SNR_WIN); MAG_AUTO, MAGERR_AUTO and FLUX_RADIUS are in both. +# SExtractor detection columns, from the tile SExtractor run (tile_detect). MAG_AUTO MAGERR_AUTO MAG_WIN diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index f105d23b4..9909eb776 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -41,18 +41,19 @@ The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor run that the shape chain does not need co-scheduled, and folding it in would add its runtime to a sum that has no room. -``tile_detection: unions_catalogue`` (config.yaml) replaces that SExtractor run -with two rules: tile_get_catalogue fetches the UNIONS per-tile catalogue and -its r-band segmentation map (get_images_runner, config_tile_Gic.ini) and -tile_detect converts it to the FITS-LDAC sexcat SExtractor would have written -(read_ext_sexcat_runner, config_tile_Uc.ini), with the tile image's header, -VIGNET stamps cut from the tile image with neighbours' footprints set to -1e30 -as SExtractor sets them (so ngmix masks neighbours in both modes), and the -multi-epoch post-processing. It keeps the catalogue's own -NUMBER, from which make_cat builds ``TILE_UNIQUE_ID`` exactly as in SExtractor -mode. The converter writes run_sp_tile_Sx/read_ext_sexcat_runner, and the rule links it as -sextractor_runner, the one path every downstream config reads; the manifest is -tile_detect.json in both modes, so tile_vignets onwards is the same DAG. +tile_detect runs SExtractor on the tile image (sextractor_runner, +config_tile_Sx.ini) for data and image simulations alike, so both get the same +detection, windowed positions, VIGNET neighbour marking and columns. +``tile_detection: unions_catalogue`` (config.yaml, the data default) adds one +rule and one step: tile_get_catalogue fetches the UNIONS per-tile catalogue +(get_images_runner, config_tile_Gic.ini), and tile_detect joins its SExtractor +rows to it before the multi-epoch post-processing (MATCH_CATALOGUE, set +through SP_MATCH_CATALOGUE). Each row pairs with its mutual nearest catalogue +object within 1 px and takes its NUMBER, from which make_cat builds +``TILE_UNIQUE_ID``, the key shared with the photometry and photo-z +catalogues; unpaired rows leave the catalogue. The catalogue is SExtractor run +by MegaPipe on the same DR6 image with the same configuration, so the join is +exact; on other pixels (a DR5 image) it fails loudly. There is no `tile_mask` rule, and there will not be one (PR #847). ShapePipe generates no masks: tiles have no instrument flag image of their own, so @@ -472,36 +473,13 @@ rule tile_merge_headers: shell: sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") -# The tile's galaxy sample. One of two definitions of tile_detect, chosen at -# parse time by the run config; both produce manifests/tile_detect.json and -# run_sp_tile_Sx/sextractor_runner/output/sexcat.fits. -if TILE_DETECTION == "sextractor": +# The UNIONS per-tile catalogue (CFIS..r.cat), from a local mirror or +# from vos, the way tile_get_images fetches the image; tile_detect joins its +# SExtractor detections to it. Reads only tile_numbers.txt, which unit_pre +# writes; the edge on the image manifest is what puts it after the prepare +# phase. +if TILE_DETECTION == "unions_catalogue": - # SExtractor object detection on the tile. - rule tile_detect: - input: - uz = f"{TILE_DIR}/manifests/tile_uncompress.json", - mh = rules.tile_merge_headers.output.manifest, - output: - manifest = f"{TILE_DIR}/manifests/tile_detect.json" - log: - f"{TILE_DIR}/logs/tile_detect.json" - params: - pre = lambda wc: unit_pre("tile_detect", wc.tile), - script_hash = SCRIPT_HASH - threads: 8 - resources: - mem_mb = lambda wc, attempt: 16000 * attempt, - runtime = 180 - shell: - sp_shell("tile_detect", "config_tile_Sx.ini") - -else: - - # Fetch the UNIONS per-tile catalogue (CFIS..r.cat) and segmentation - # map (CFIS..r.seg.fits.fz), from a local mirror or from vos, the way tile_get_images fetches the image. Reads - # only tile_numbers.txt, which unit_pre writes; the edge on the image - # manifest is what puts it after the prepare phase. rule tile_get_catalogue: input: git = f"{TILE_DIR}/manifests/tile_get_images.json", @@ -523,36 +501,45 @@ else: shell: sp_shell("tile_get_catalogue", "config_tile_Gic.ini") - # Convert the catalogue to the FITS-LDAC sexcat the chain expects. The - # completeness check counts read_ext_sexcat_runner's own output; the link - # after it is what makes that output readable at the sextractor_runner path - # of every downstream config, and it is made only on success so a failed - # run leaves nothing at that path. - rule tile_detect: - input: - cat = rules.tile_get_catalogue.output.manifest, - git = f"{TILE_DIR}/manifests/tile_get_images.json", - mh = rules.tile_merge_headers.output.manifest, - output: - manifest = f"{TILE_DIR}/manifests/tile_detect.json" - log: - f"{TILE_DIR}/logs/tile_detect.json" - params: - pre = lambda wc: unit_pre( - "tile_detect", wc.tile, - env={"SP_TILE_DETECTION": TILE_DETECTION}), - script_hash = SCRIPT_HASH - threads: 1 - resources: - # The whole tile image plus one 51x51 float32 stamp per object. - mem_mb = lambda wc, attempt: 8000 * attempt, - runtime = 60 - shell: - sp_shell("tile_detect", "config_tile_Uc.ini", - post="if [ $rc -eq 0 ]; then\n" - ' ln -s read_ext_sexcat_runner ' - '"$SP_RUN/output/run_sp_tile_Sx/sextractor_runner"\n' - "fi\n") + +# tile_detect's inputs: the catalogue manifest too when it joins one. +DETECT_INPUTS = {"uz": f"{TILE_DIR}/manifests/tile_uncompress.json", + "mh": rules.tile_merge_headers.output.manifest} +if TILE_DETECTION == "unions_catalogue": + DETECT_INPUTS["cat"] = rules.tile_get_catalogue.output.manifest + + +def detect_env(tile): + """tile_detect's prologue exports: the catalogue to join, if any. + + Nothing under tile_detection: sextractor, so that prologue (a rerun + trigger) is the one the image simulations have always run with. + """ + if TILE_DETECTION != "unions_catalogue": + return {} + gic = f"{tile_dir(tile)}/output/run_sp_tile_Gic/get_images_runner/output" + return {"SP_MATCH_CATALOGUE": f"{gic}/CFIS_cat{unit_num(tile)}.cat"} + + +# SExtractor object detection on the tile; under unions_catalogue joined to +# the UNIONS catalogue (config_tile_Sx.ini, MATCH_CATALOGUE). +rule tile_detect: + input: + **DETECT_INPUTS + output: + manifest = f"{TILE_DIR}/manifests/tile_detect.json" + log: + f"{TILE_DIR}/logs/tile_detect.json" + params: + pre = lambda wc: unit_pre("tile_detect", wc.tile, + env=detect_env(wc.tile)), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 + shell: + sp_shell("tile_detect", "config_tile_Sx.ini") # Configured PSF interpolation to galaxies + vignet postage stamps: the last # stage that reads exposure products, and the bulk intra-tile intermediate. The store it @@ -1040,7 +1027,6 @@ rule final_cat_merge: tile_list = str(config["tile_list"]), index_db = str(INDEX_DB), param_file = str(CONFIG_DIR / "final_cat.param"), - tile_detection = TILE_DETECTION, campaign = CAMPAIGN, snapshot = str(SNAPSHOT_JSON), inputs = unit_fingerprint(TILES_READY), @@ -1065,5 +1051,4 @@ rule final_cat_merge: " --output {output.merged}" " --campaign '{params.campaign}'" " --param-file '{params.param_file}'" - " --tile-detection {params.tile_detection}" " --snapshot-json '{params.snapshot}'" diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 52720b724..b39a951ed 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -72,15 +72,13 @@ import sys from pathlib import Path -# How the tile's galaxy sample is produced: SExtractor on the tile image, or -# the UNIONS per-tile catalogue converted in place. The run config's -# `tile_detection:` picks one; the rules export it as $SP_TILE_DETECTION only -# when it is not the default, so a SExtractor run's prologue is unchanged. +# How the tile's galaxy sample is defined: SExtractor on the tile image alone, +# or those detections joined to the UNIONS per-tile catalogue, whose NUMBER +# they take. The run config's `tile_detection:` picks one. TILE_DETECTIONS = ("sextractor", "unions_catalogue") # stage -> {runner_subdir: {expect, [warn], [subpath]}} -# exp_psf and tile_vignets are selected by $SP_PSF at check time, tile_detect -# by $SP_TILE_DETECTION. +# exp_psf and tile_vignets are selected by $SP_PSF at check time. # @sc [decision:per_unit_completeness] COMPLETENESS = { # --- tile prepare (phase A) --- @@ -142,13 +140,9 @@ # --- tile post --- "tile_merge_headers": {"merge_headers_runner": dict(expect=1)}, - # The fetched UNIONS catalogue and its r-band segmentation map. - "tile_get_catalogue": {"get_images_runner": dict(expect=2)}, - "tile_detect": { - "sextractor": {"sextractor_runner": dict(expect=2)}, - # The FITS-LDAC sexcat converted from the fetched catalogue. - "unions_catalogue": {"read_ext_sexcat_runner": dict(expect=1)}, - }, + # The fetched UNIONS catalogue. + "tile_get_catalogue": {"get_images_runner": dict(expect=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2)}, "tile_vignets": { "psfex": { "psfex_interp_runner": dict(expect=1), @@ -223,13 +217,6 @@ def check_counts(stage, run_dir): raise ValueError( f"Invalid SP_PSF={psf_model!r}; expected one of {sorted(table)}." ) from exc - elif stage == "tile_detect": - detection = os.environ.get("SP_TILE_DETECTION", TILE_DETECTIONS[0]) - if detection not in TILE_DETECTIONS: - raise ValueError( - f"Invalid SP_TILE_DETECTION={detection!r}; expected one of " - f"{', '.join(TILE_DETECTIONS)}.") - table = table[detection] details, ok = [], True for runner, spec in table.items(): n = count_products(run_dir, runner, spec) @@ -262,8 +249,6 @@ def check_counts(stage, run_dir): "exp_psf": ("exp", "run_sp_exp_SxSePsf"), "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), "tile_get_catalogue": ("tile", "run_sp_tile_Gic"), - # Both detection modes write here (config_tile_Sx.ini / config_tile_Uc.ini - # share the RUN_NAME): the chain downstream reads one path. "tile_detect": ("tile", "run_sp_tile_Sx"), "tile_vignets": ("tile", "run_sp_tile_PiViVi"), "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), From 2a1fd491e975f4f06310cb03311a56130fdcadb1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:10:19 +0200 Subject: [PATCH 32/44] sextractor_runner: single-image SExtractor without a detection image Without DETECTION_IMAGE the runner called SExtractor in dual-image mode with the same image and weight twice, which changes the background, MAG_AUTO and FLUX_RADIUS of a fifth of a tile's objects against the single-image call and the UNIONS catalogue. Dual-image mode stays for a real detection image. Closes #936 Co-Authored-By: Claude Opus 5.5 --- .../sextractor_package/sextractor_script.py | 41 ++++++---- tests/module/test_sextractor_command_line.py | 81 +++++++++++++++++++ 2 files changed, 106 insertions(+), 16 deletions(-) create mode 100644 tests/module/test_sextractor_command_line.py diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index 5b97fb07d..c3716db74 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -384,6 +384,13 @@ def set_input_files( Set up all of the input image files. + Without a detection image, SExtractor runs in single-image mode on + the measurement image: ``img -WEIGHT_IMAGE w``. With one, it runs in + dual-image mode, ``det,img -WEIGHT_IMAGE det_w,w``. Dual-image mode + with the same image twice is not equivalent to single-image mode: it + changes the background and the photometry of a fifth of the objects of + a UNIONS tile (#936). + @sc [decision:detection.weight_map_usage,label:convention] weight-map-on-both-images With a weight file, the same map weights detection and measurement (a separate detection weight only when DETECTION_WEIGHT is set); without @@ -443,26 +450,28 @@ def set_input_files( "DETECTION_WEIGHT cannot be True " + "if WEIGHT_FILE is False" ) - # Check for separate image file for detection and measurement + # A separate detection image switches SExtractor to dual-image mode; + # without one, the measurement image alone (single-image mode). if use_detect_img: - self._detect_img_path = self._all_input_path[extra] + self._image_arg = ( + f"{self._all_input_path[extra]},{self._meas_img_path}" + ) extra += 1 else: - self._detect_img_path = self._meas_img_path + self._image_arg = self._meas_img_path - # Check for separate weight file corresponding to the detection image. - # If False, use measurement weight image. - # Note: This could be changed, and no weight image could be used, but - # this might lead to more user errors. + # In dual-image mode the detection image takes the detection weight + # if given, else the measurement weight. if use_weight: - if use_detect_weight: - detect_weight_path = self._all_input_path[extra] - extra += 1 - else: - detect_weight_path = weight_image - self._cmd_line_extra += ( - f" -WEIGHT_IMAGE {detect_weight_path}" + f",{weight_image}" - ) + weight_arg = weight_image + if use_detect_img: + if use_detect_weight: + detect_weight_path = self._all_input_path[extra] + extra += 1 + else: + detect_weight_path = weight_image + weight_arg = f"{detect_weight_path},{weight_image}" + self._cmd_line_extra += f" -WEIGHT_IMAGE {weight_arg}" else: self._cmd_line_extra += " -WEIGHT_TYPE None" @@ -563,7 +572,7 @@ def make_command_line(self, exec_path): """ # Base arguments for SExtractor command_line_base = ( - f"{exec_path} {self._detect_img_path},{self._meas_img_path} " + f"{exec_path} {self._image_arg} " + f"-c {self._path_dot_sex} " + f"-PARAMETERS_NAME {self._path_dot_param} " + f"-FILTER_NAME {self._path_dot_conv} " diff --git a/tests/module/test_sextractor_command_line.py b/tests/module/test_sextractor_command_line.py new file mode 100644 index 000000000..47488df0d --- /dev/null +++ b/tests/module/test_sextractor_command_line.py @@ -0,0 +1,81 @@ +"""SExtractor command line built by ``SExtractorCaller``. + +Without a detection image the call is single-image (``img -WEIGHT_IMAGE w``); +dual-image mode (``det,img -WEIGHT_IMAGE det_w,w``) only with a detection +image, since dual-image mode on the same image twice changes the measurements. +""" + +import shlex + +import pytest + +from shapepipe.modules.sextractor_package.sextractor_script import ( + SExtractorCaller, +) + + +def _command(inputs, **flags): + options = dict( + use_weight=False, use_flag=False, use_psf=False, + use_detection_image=False, use_detection_weight=False, + use_zero_point=False, use_background=False, + ) + options.update(flags) + caller = SExtractorCaller( + inputs, "/out", "-000-000", "d.sex", "d.param", "d.conv", + check_image=[""], **options, + ) + argv = shlex.split(caller.make_command_line("source-extractor")) + options = {argv[i]: argv[i + 1] for i in range(2, len(argv) - 1) + if argv[i].startswith("-")} + return argv, options + + +def test_tile_single_image_with_weight(): + argv, opt = _command(["img.fits", "w.fits"], use_weight=True) + assert argv[:2] == ["source-extractor", "img.fits"] + assert opt["-WEIGHT_IMAGE"] == "w.fits" + assert "-FLAG_IMAGE" not in opt + assert opt["-CATALOG_NAME"] == "/out/sexcat-000-000.fits" + assert opt["-CHECKIMAGE_TYPE"] == "NONE" + + +def test_exposure_single_image_with_weight_and_flag(): + argv, opt = _command(["img.fits", "w.fits", "f.fits"], + use_weight=True, use_flag=True) + assert argv[1] == "img.fits" + assert opt["-WEIGHT_IMAGE"] == "w.fits" + assert opt["-FLAG_IMAGE"] == "f.fits" + + +def test_single_image_without_weight(): + argv, opt = _command(["img.fits"]) + assert argv[1] == "img.fits" + assert opt["-WEIGHT_TYPE"] == "None" + assert "-WEIGHT_IMAGE" not in opt + + +@pytest.mark.parametrize("detection_weight", [False, True]) +def test_dual_image_with_detection_image(detection_weight): + inputs = ["img.fits", "w.fits", "det.fits"] + if detection_weight: + inputs.append("det_w.fits") + argv, opt = _command(inputs, use_weight=True, use_detection_image=True, + use_detection_weight=detection_weight) + assert argv[1] == "det.fits,img.fits" + det_w = "det_w.fits" if detection_weight else "w.fits" + assert opt["-WEIGHT_IMAGE"] == f"{det_w},w.fits" + + +def test_check_images_named_per_unit(): + caller = SExtractorCaller( + ["img.fits", "w.fits"], "/out", "-001-002", "d.sex", "d.param", + "d.conv", True, False, False, False, False, False, False, + check_image=["BACKGROUND", "SEGMENTATION"], + ) + argv = shlex.split(caller.make_command_line("source-extractor")) + assert argv[1] == "img.fits" + assert argv[argv.index("-CHECKIMAGE_TYPE") + 1] == "BACKGROUND,SEGMENTATION" + assert argv[argv.index("-CHECKIMAGE_NAME") + 1] == ( + "/out/background-001-002.fits,/out/segmentation-001-002.fits" + ) From 7ac98844e6d7d8e9742c0229571ee7dd3cf4f364 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:10:20 +0200 Subject: [PATCH 33/44] match_catalogue: two-sided pairing guard; record the eight-tile validation The run also stops when fewer than MATCH_MIN_FRACTION of the catalogue objects pair, which catches a detection configuration that finds fewer objects than the catalogue. Tests cover the radius bound and both sides of the guard. astra.yaml records the eight-tile numbers, single-image detection, and the MAG_AUTO differences beyond a pixel-stack overflow. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 36 +++++++++++++------ .../sextractor_package/match_catalogue.py | 36 +++++++++++++------ tests/module/test_match_catalogue.py | 26 +++++++++++++- workflow/config/cfis/config_tile_Sx.ini | 7 ++-- workflow/config/cfis/default_tile.sex | 1 + 5 files changed, 82 insertions(+), 24 deletions(-) diff --git a/astra.yaml b/astra.yaml index 2c58269a0..a0cbb3ae1 100644 --- a/astra.yaml +++ b/astra.yaml @@ -373,14 +373,18 @@ analyses: object list and one ID (TILE_UNIQUE_ID); unpaired detections leave the catalogue. The catalogue is MegaPipe's SExtractor run on the same DR6 image with the configuration of default_tile.sex - (detection_threshold_policy), so on tile 186.307 every one of the - 41,201 detections pairs, at a median separation of 4e-5 px. The 45 - catalogue objects with no detection (0.11%; 4 in 20 < MAG_AUTO < - 24.5) are faint deblended children inside bright parents, which - SExtractor 2.25 leaves merged; they have photometry but no shape. - The join needs the tile image to be the catalogue's release: on - the DR5 image of the same tile only ~95% of detections pair, and - below 99% the run stops. PSF stars come from the exposure-level + (detection_threshold_policy), so pairs agree to 4e-5 px at the + median. On tile 186.307 every one of the 41,201 detections pairs; + over eight DR6 tiles spanning a rich cluster, a bright star's halo, + low latitude and the survey edge, at least 99.6% of detections and + 99.4% of catalogue objects pair. The catalogue objects with no + detection (0.1-0.6%, mostly fainter than MAG_AUTO 24.5) are + deblended children that ShapePipe's SExtractor leaves merged, + concentrated around very large objects; they have photometry but + no shape. The join needs the tile image to be the catalogue's + release: on the DR5 image of a tile only ~95% of detections pair. + The run stops when fewer than 99% of the detections, or of the + catalogue objects, pair. PSF stars come from the exposure-level SExtractor run in both options. Image simulations, which have no UNIONS catalogue, keep SExtractor's own NUMBER. Values: @@ -401,7 +405,13 @@ analyses: catalogue whose NUMBER committed data adopts (tile_detection). Run on the DR6 image of tile 186.307, it reproduces the DR6 catalogue to 99.89% completeness and 100% purity with identical segmentation pixels; the 45 misses - are faint deblended children of bright parents (#929). This + are faint deblended children of bright parents (#929). On tiles + where SExtractor's pixel stack overflows on a very large object (a + cD galaxy, a bright star's halo, at MEMORY_PIXSTACK = 300000), + MAG_AUTO and FLUX_RADIUS differ from the catalogue by more than 1% + for 2-11% of objects, most of them beyond the overflow in scan + order; a larger stack moves but does not remove the difference, so + the catalogue's run differs in a setting not recorded here. This validates the configuration on data pixels, not detection on simulated images. The 7x7 Gaussian of FWHM 3 px is near the CFIS average seeing of 0.65 arcsec (about 3.5 px at @@ -415,6 +425,7 @@ analyses: default_tile.sex#ANALYSIS_THRESH = 1.0; default_tile.sex#DETECT_MINAREA = 3; default_tile.sex#FILTER = Y; + default_tile.sex#MEMORY_PIXSTACK = 300000; config_tile_Sx.ini#SEXTRACTOR_RUNNER.DOT_CONV_FILE = $SP_CONFIG/gauss_3.0_7x7.conv; config_exp_psfex.ini#SEXTRACTOR_RUNNER.DOT_SEX_FILE = @@ -625,7 +636,12 @@ analyses: column list: tiles have no instrument flag image and no detection coadd exists, so detection sees every tile pixel and the tile catalogue carries no IMAFLAGS_ISO; final_cat.param, the merge's - exact allow-list, does not request it. + exact allow-list, does not request it. Without a detection image + sextractor_runner calls SExtractor in single-image mode, which on + tile 186.307 reproduces the UNIONS catalogue's MAG_AUTO to 0.01 mag + for all but 0.03% of objects. Dual-image mode with the same image + twice is not equivalent: it changes the background, MAG_AUTO and + FLUX_RADIUS of a fifth of the objects (#936). Values: SEXTRACTOR_RUNNER.DETECTION_IMAGE = False; SEXTRACTOR_RUNNER.FLAG_IMAGE = False. diff --git a/src/shapepipe/modules/sextractor_package/match_catalogue.py b/src/shapepipe/modules/sextractor_package/match_catalogue.py index 23f4c5474..a6b70f8d4 100644 --- a/src/shapepipe/modules/sextractor_package/match_catalogue.py +++ b/src/shapepipe/modules/sextractor_package/match_catalogue.py @@ -6,8 +6,9 @@ The UNIONS tile catalogue (Stephen Gwyn's MegaPipe SExtractor run on the DR6 tile) defines the object list and the ``NUMBER`` that the photometry and photo-z catalogues share. ShapePipe runs its own SExtractor on the same image -with the same detection configuration, which reproduces that catalogue object -for object, and takes membership and ``NUMBER`` from it here: every +with the same detection configuration, which reproduces that catalogue for +all but a few objects in a thousand, and takes membership and ``NUMBER`` from +it here: every measurement (windowed positions, ``VIGNET`` and its neighbour marking, the SExtractor columns) stays SExtractor's own. @@ -103,11 +104,16 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, marked ``UNMATCHED_LABEL``. The catalogue is rewritten in place; every other HDU and column is kept. - The join is exact when both catalogues come from the same pixels: on the - DR6 image of tile 186.307 every SExtractor row pairs, at a median - separation of 4e-5 pixel. Run on different pixels (a DR5 image against - the DR6 catalogue) only ~95% pair, so a pairing fraction below - ``min_fraction`` stops the run rather than silently losing objects. + Both catalogues come from the same pixels, so pairs agree to ~1e-4 + pixel. On eight DR6 tiles at least 99.6% of SExtractor rows and 99.4% of + UNIONS objects pair; the shortfall is SExtractor deblending differently + near very large objects (a cD galaxy, a bright star's halo). Run on + different pixels (a DR5 image against the DR6 catalogue) only ~95% pair. + So the run stops when fewer than ``min_fraction`` of the SExtractor rows, + or of the external objects, pair: the first catches a catalogue from + other pixels, the second a detection configuration that finds fewer + objects than the catalogue's, which would leave its objects without + shapes. Parameters ---------- @@ -119,8 +125,8 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, radius : float, optional Largest separation of a pair, in pixels; default 1 min_fraction : float, optional - Smallest acceptable fraction of SExtractor rows with a partner; - default 0.99 + Smallest acceptable fraction of SExtractor rows, and of external + objects, with a partner; default 0.99 w_log : logging.Logger, optional Pipeline logger @@ -134,7 +140,8 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, Raises ------ ValueError - If fewer than ``min_fraction`` of the SExtractor rows pair + If fewer than ``min_fraction`` of the SExtractor rows, or of the + external objects, pair @sc [decision:detection.tile_detection] """ @@ -164,6 +171,15 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, + " same pixels as the image: check that the tile image is the" + " release of the catalogue (DR6)." ) + fraction_ext = len(i_sex) / max(len(ext), 1) + if fraction_ext < min_fraction: + raise ValueError( + f"Only {fraction_ext:.4f} of the objects of {ext_cat_path} pair" + + f" with a row of {cat_path} within {radius} px (minimum" + + f" {min_fraction}; {summary}). SExtractor finds fewer objects" + + " than the catalogue: check that the detection configuration" + + " is the catalogue's." + ) old_number = np.asarray(data["NUMBER"]) new_number = np.full(len(data), UNMATCHED_LABEL, np.int64) diff --git a/tests/module/test_match_catalogue.py b/tests/module/test_match_catalogue.py index 84105ea16..2d6dedb21 100644 --- a/tests/module/test_match_catalogue.py +++ b/tests/module/test_match_catalogue.py @@ -67,7 +67,7 @@ def test_rows_take_the_external_number_one_to_one(tmp_path): ext = _external(tmp_path / "ext.cat", numbers + [1006], list(X[order] + 1e-4) + [300.0], list(Y[order] - 1e-4) + [300.0]) - counts = mc.match_catalogue(cat, ext) + counts = mc.match_catalogue(cat, ext, min_fraction=0.8) data = _objects(cat) expected = np.empty(5, int) expected[order] = numbers @@ -104,6 +104,30 @@ def test_pairs_are_mutual_nearest_neighbours(tmp_path): np.testing.assert_allclose(data["X_IMAGE"], [10.0, 90.0]) +def test_pairs_lie_within_the_radius(tmp_path): + """Mutual nearest neighbours 2 px apart, beyond the radius, do not pair.""" + x = np.append(X, 300.0) + y = np.append(Y, 300.0) + cat = _sexcat(tmp_path / "sexcat.fits", x, y) + ext = _external(tmp_path / "ext.cat", [1, 2, 3, 4, 5, 6], + np.append(X, 302.0), np.append(Y, 300.0)) + counts = mc.match_catalogue(cat, ext, radius=1.0, min_fraction=0.8) + assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4, 5] + assert counts["n_dropped"] == 1 and counts["n_external_only"] == 1 + + +def test_too_few_catalogue_objects_paired_stop_the_run(tmp_path): + """Fewer detections than catalogue objects (a drifted detection + configuration) fails, though every detection pairs.""" + cat = _sexcat(tmp_path / "sexcat.fits", X[:4], Y[:4]) + ext = _external(tmp_path / "ext.cat", [1, 2, 3, 4, 5], X, Y) + with pytest.raises(ValueError, match="fewer objects"): + mc.match_catalogue(cat, ext, min_fraction=0.99) + assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4] + counts = mc.match_catalogue(cat, ext, min_fraction=0.8) + assert counts["n_paired"] == 4 and counts["n_external_only"] == 1 + + def test_too_few_pairs_stop_the_run(tmp_path): """Below MATCH_MIN_FRACTION (other pixels: the DR5 image) the run fails.""" cat = _sexcat(tmp_path / "sexcat.fits", X, Y) diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index b73b07152..524635054 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -131,9 +131,10 @@ MATCH_CATALOGUE = ${SP_MATCH_CATALOGUE:-} # @sc [decision:detection.tile_detection] MATCH_RADIUS = 1.0 -# Smallest fraction of SExtractor rows that must pair; fewer stops the run. -# A catalogue measured on other pixels (the DR5 image of a tile against its -# DR6 catalogue) pairs ~95%. +# Smallest fraction of SExtractor rows, and of catalogue objects, that must +# pair; fewer stops the run. DR6 tiles pair at least 99.6% of rows and 99.4% +# of objects (the lowest near a cD galaxy or a bright star); a catalogue on other +# pixels (the DR5 image of a tile against its DR6 catalogue) pairs ~95%. # @sc [decision:detection.tile_detection] MATCH_MIN_FRACTION = 0.99 diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex index ca7367c3e..dd44248aa 100644 --- a/workflow/config/cfis/default_tile.sex +++ b/workflow/config/cfis/default_tile.sex @@ -121,6 +121,7 @@ BACK_FILTTHRESH 0.0 # Threshold above which the background- #--------------------- Memory (change with caution!) ------------------------- MEMORY_OBJSTACK 3000 # number of objects in stack +# @sc [decision:detection.detection_threshold_policy] MEMORY_PIXSTACK 300000 # number of pixels in stack MEMORY_BUFSIZE 1024 # number of lines in buffer From e02f1f8124d8e2a2ac5447c7ae6e721cbaf864f1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:10:20 +0200 Subject: [PATCH 34/44] Config expansion rejects malformed ${ forms ${VAR-x}, ${VAR:=x} and ${ VAR } passed through as literal paths. Co-Authored-By: Claude Opus 5.5 --- src/shapepipe/pipeline/config.py | 9 +++++++- tests/unit/test_config_parse.py | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/shapepipe/pipeline/config.py b/src/shapepipe/pipeline/config.py index 64d2e0592..2c294ae65 100644 --- a/src/shapepipe/pipeline/config.py +++ b/src/shapepipe/pipeline/config.py @@ -38,9 +38,16 @@ def _expandvars_strict(value): Raises ------ ValueError - If a referenced environment variable without a default is not set + If a referenced environment variable without a default is not set, + or a ``${`` opens none of the forms above (``${VAR-x}``, + ``${VAR:=x}``, ``${ VAR }``) """ + if "${" in _VAR_RE.sub("", value): + raise ValueError( + f"Config value '{value}' has a '${{' that is not $VAR, ${{VAR}}" + + " or ${VAR:-default}." + ) unset = [] def substitute(match): diff --git a/tests/unit/test_config_parse.py b/tests/unit/test_config_parse.py index 4aff1efbf..458448ac3 100644 --- a/tests/unit/test_config_parse.py +++ b/tests/unit/test_config_parse.py @@ -251,3 +251,38 @@ def test_module_sections_check_catches_a_missing_repeated_section(): problems = _missing_required_keys(parser) assert any("VIGNETMAKER_RUNNER_RUN_1" in problem for problem in problems) + + +def _expanded(value): + from shapepipe.pipeline.config import CustomParser + + parser = CustomParser() + parser.read_dict({"S": {"K": value}}) + return parser.getexpanded("S", "K") + + +def test_env_default_applies_when_unset_or_empty(monkeypatch): + """${VAR:-x} is x when VAR is unset and when it is empty, as in the shell.""" + monkeypatch.delenv("SP_TEST_VAR", raising=False) + assert _expanded("${SP_TEST_VAR:-x}") == "x" + monkeypatch.setenv("SP_TEST_VAR", "") + assert _expanded("${SP_TEST_VAR:-x}") == "x" + assert _expanded("${SP_TEST_VAR:-}") == "" + assert _expanded("${SP_TEST_VAR}") == "" + + +def test_env_value_is_inserted_unexpanded(monkeypatch): + """A value holding '$' or '${' is inserted as is, not expanded again.""" + monkeypatch.setenv("SP_TEST_VAR", "/a/$HOME/${X:=y}") + assert _expanded("$SP_TEST_VAR/b") == "/a/$HOME/${X:=y}/b" + assert _expanded("${SP_TEST_VAR:-z}") == "/a/$HOME/${X:=y}" + + +@pytest.mark.parametrize( + "value", ["${SP_TEST_VAR-x}", "${SP_TEST_VAR:=x}", "${ SP_TEST_VAR }", + "/p/${SP_TEST_VAR"]) +def test_malformed_brace_is_rejected(monkeypatch, value): + """A '${' that is not $VAR, ${VAR} or ${VAR:-x} fails, set or not.""" + monkeypatch.setenv("SP_TEST_VAR", "v") + with pytest.raises(ValueError, match="not \\$VAR"): + _expanded(value) From c6725a7c455f99a91cd27e07c930b57a075c988b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:10:20 +0200 Subject: [PATCH 35/44] tile_detect: 1 core, 4 GB, 20 min from eight measured tiles Peak RSS 1.75 GiB and wall time 143 s at most; SExtractor is built without threads. One retry doubles memory and runtime. Image simulations export SP_MATCH_CATALOGUE empty. Stale comments on the catalogue fetch. Co-Authored-By: Claude Opus 5.5 --- tests/workflow/test_dag.py | 4 ++-- workflow/config.yaml | 3 +-- workflow/rules/tile.smk | 35 +++++++++++++++++++++++++--------- workflow/scripts/run_report.py | 2 +- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index 69a70d17c..dd5fc7f70 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -33,7 +33,7 @@ def test_rule_set_matches_input_mode(campaign, dag): def test_tile_detect_joins_the_catalogue_iff_unions(campaign, dag): """Data's tile_detect waits on the fetch and exports its catalogue as - SP_MATCH_CATALOGUE; the image-simulation prologue exports nothing new.""" + SP_MATCH_CATALOGUE; the image-simulation prologue exports it empty.""" for job in dag.jobs_for("tile_detect"): tile = job.wildcards.tile inputs = {str(f) for f in job.input} @@ -46,7 +46,7 @@ def test_tile_detect_joins_the_catalogue_iff_unions(campaign, dag): assert f"export SP_MATCH_CATALOGUE='{cat}'" in job.params.pre else: assert fetch not in inputs - assert "SP_MATCH_CATALOGUE" not in job.params.pre + assert "export SP_MATCH_CATALOGUE=''" in job.params.pre.split("\n") def test_clean_exposure_waits_on_persist_iff_psf(campaign, dag): diff --git a/workflow/config.yaml b/workflow/config.yaml index 8f2cd90c1..8b36c70fa 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -63,8 +63,7 @@ machines: inputs: tiles: $base_dir/unions-wl/tiles exposures: $base_dir/unions-wl/exposures - # UNIONS per-tile catalogues and segmentation maps (CFIS..r.cat, - # CFIS..r.seg.fits.fz), fetched by vcp in + # UNIONS per-tile catalogues (CFIS..r.cat), fetched by vcp in # the job: needs network from compute nodes and ~/.ssl/cadcproxy.pem. catalogues: vos:cfis/tiles_DR6 outputs: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 9909eb776..cc70e96b0 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -37,9 +37,8 @@ Distinct tiles share no edge, so each is one group job per tile: job reaches the widest partition set (plus cpubackfill) — which is why each member's runtime is measured p99 plus margin and not the old ceiling. -The heavy middle (tile_detect) stays out: it is a 16 GB / 8 thread SExtractor -run that the shape chain does not need co-scheduled, and folding it in would add -its runtime to a sum that has no room. +tile_detect stays out: the shape chain does not need it co-scheduled, and +folding it in would add its runtime to a sum that has no room. tile_detect runs SExtractor on the tile image (sextractor_runner, config_tile_Sx.ini) for data and image simulations alike, so both get the same @@ -512,11 +511,11 @@ if TILE_DETECTION == "unions_catalogue": def detect_env(tile): """tile_detect's prologue exports: the catalogue to join, if any. - Nothing under tile_detection: sextractor, so that prologue (a rerun - trigger) is the one the image simulations have always run with. + Empty under tile_detection: sextractor (image simulations), so that no + value left exported in the submitting shell reaches the job. """ if TILE_DETECTION != "unions_catalogue": - return {} + return {"SP_MATCH_CATALOGUE": ""} gic = f"{tile_dir(tile)}/output/run_sp_tile_Gic/get_images_runner/output" return {"SP_MATCH_CATALOGUE": f"{gic}/CFIS_cat{unit_num(tile)}.cat"} @@ -534,10 +533,28 @@ rule tile_detect: pre = lambda wc: unit_pre("tile_detect", wc.tile, env=detect_env(wc.tile)), script_hash = SCRIPT_HASH - threads: 8 + # One core: the container's SExtractor is built without threads + # (NTHREADS 4 warns and runs in the same 36 s), default_tile.sex sets + # NTHREADS 1, the join and the post-processing are serial Python, and + # `-b {threads}` only sets the SMP batch over input sets, of which a tile + # is one. + # + # MEASURED on eight DR6 tiles chosen to span conditions (high latitude, + # b = 18 deg, the A2199 cluster, Alioth's halo, two survey-edge tiles at + # 96% zero weight; 1-10 exposures), the step as shapepipe_run runs it on + # candide: wall time 23-143 s (SExtractor + join <= 50 s, post-processing + # <= 92 s), peak RSS 1.75 GiB, set by the join holding the ~430 MB + # catalogue twice. Image simulations run the same step on tiles of the same + # size (10000 x 10000 px); SExtractor alone on one takes 34 s and 0.74 GiB. + # 4000 MB is 2.2x the worst tile, and an OOM retries at 8000. The runtime + # is ~8x the slowest tile, for /scratch I/O on nibi (a 400 MB image and + # weight in, a ~430 MB catalogue out). nibi bills max(cores, mem_GB/4), so + # the job bills 1 core-equivalent. + threads: 1 + retries: 1 resources: - mem_mb = lambda wc, attempt: 16000 * attempt, - runtime = 180 + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = lambda wc, attempt: 20 * attempt shell: sp_shell("tile_detect", "config_tile_Sx.ini") diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index a8efe82ae..bd846d34d 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -60,7 +60,7 @@ TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", "tile_merge_headers", "tile_detect", "tile_vignets", "tile_ngmix", "tile_merge_cats", "tile_make_cat"] -# The catalogue fetch exists only when the run converts the UNIONS catalogue +# The catalogue fetch exists only when the run joins the UNIONS catalogue # (config.yaml's tile_detection); the callers pass the mode in the environment # so a SExtractor run does not report the stage as not run. if os.environ.get("SP_TILE_DETECTION") == "unions_catalogue": From 5779c0749e1057155d3dcbbb45d0b99790bfa92d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:10:29 +0200 Subject: [PATCH 36/44] Update the params pin for this campaign boundary Co-Authored-By: Claude Opus 5.5 --- tests/workflow/params_pin.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index 4502a6f20..6de487277 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -11,7 +11,7 @@ "final_cat_merge": "e7f46859c4503a2220713d7bb2507555515d0a9632d780b20f14c59e32210023", "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", - "tile_detect": "530a89d61ca61f42d660d2b1d9c1c9926d9b552618bb1a131922b2473f5aa9f2", + "tile_detect": "8336b148769e9d43b64f0d945c05e7d163c64dba4e6287e8c1964d1502e3ffe1", "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", @@ -24,7 +24,7 @@ "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "ce48433c4ecce26c999d09705861eeb05625e09ad5bf3383213192729fc1f2c7", + "sha256": "d67ee5da728e8fee513e1f01a522b386eb6a86d21c9a36250afd6013a06301d5", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", From 6a8ee5000a4bd3a22cf9a7f46f8bcc00fec5623c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:24:34 +0200 Subject: [PATCH 37/44] Tile SExtractor: MEMORY_PIXSTACK 3000000, so no object is truncated 300000 overflowed on four of eight DR6 tiles (a cD galaxy, a bright star's halo, 13 times on Alioth's tile); 3000000 clears all eight. Peak RSS rises to 1.84 GiB and wall time to 160 s on the worst tile; tile_detect's 4000 MB stays at 2.1x. The exposure config never overflows on the 40 CCDs checked and is unchanged. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 34 ++++++++++++++++----------- workflow/config/cfis/default_tile.sex | 4 +++- workflow/rules/tile.smk | 8 +++---- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/astra.yaml b/astra.yaml index a0cbb3ae1..8204587ac 100644 --- a/astra.yaml +++ b/astra.yaml @@ -376,21 +376,23 @@ analyses: (detection_threshold_policy), so pairs agree to 4e-5 px at the median. On tile 186.307 every one of the 41,201 detections pairs; over eight DR6 tiles spanning a rich cluster, a bright star's halo, - low latitude and the survey edge, at least 99.6% of detections and - 99.4% of catalogue objects pair. The catalogue objects with no - detection (0.1-0.6%, mostly fainter than MAG_AUTO 24.5) are + low latitude and the survey edge, at least 99.7% of detections and + 99.2% of catalogue objects pair. The catalogue objects with no + detection (0-0.7%, mostly fainter than MAG_AUTO 24.5) are deblended children that ShapePipe's SExtractor leaves merged, concentrated around very large objects; they have photometry but no shape. The join needs the tile image to be the catalogue's release: on the DR5 image of a tile only ~95% of detections pair. - The run stops when fewer than 99% of the detections, or of the - catalogue objects, pair. PSF stars come from the exposure-level + The run stops when the unpaired detections, or the unpaired + catalogue objects, exceed both 2% of their side and 20 objects: a + margin over deblending near large objects, a stop for other pixels. PSF stars come from the exposure-level SExtractor run in both options. Image simulations, which have no UNIONS catalogue, keep SExtractor's own NUMBER. Values: input_types.data.tile_detection = unions_catalogue; config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_RADIUS = 1.0; - config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_MIN_FRACTION = 0.99. + config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_MIN_FRACTION = 0.98; + config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_TOLERATED_UNPAIRED = 20. default: unions_catalogue options: unions_catalogue: @@ -405,13 +407,17 @@ analyses: catalogue whose NUMBER committed data adopts (tile_detection). Run on the DR6 image of tile 186.307, it reproduces the DR6 catalogue to 99.89% completeness and 100% purity with identical segmentation pixels; the 45 misses - are faint deblended children of bright parents (#929). On tiles - where SExtractor's pixel stack overflows on a very large object (a - cD galaxy, a bright star's halo, at MEMORY_PIXSTACK = 300000), - MAG_AUTO and FLUX_RADIUS differ from the catalogue by more than 1% - for 2-11% of objects, most of them beyond the overflow in scan - order; a larger stack moves but does not remove the difference, so - the catalogue's run differs in a setting not recorded here. This + are faint deblended children of bright parents (#929). The pixel + stack (MEMORY_PIXSTACK) holds the largest object on every tested + tile, a bright star's halo included, so no object is truncated. + Unexplained: on tiles with such a large object (a cD galaxy, a + bright star's halo) MAG_AUTO and FLUX_RADIUS differ from the + catalogue by more than 1% for 2-11% of objects, most of them past + the large object in SExtractor's scan order, and on one + low-latitude tile for 2% of objects spread over the tile. The + catalogue's run must differ in a setting not recorded here + (memory, version); shapes use ShapePipe's measurements and + photometry the catalogue's, so for those objects the two differ. This validates the configuration on data pixels, not detection on simulated images. The 7x7 Gaussian of FWHM 3 px is near the CFIS average seeing of 0.65 arcsec (about 3.5 px at @@ -425,7 +431,7 @@ analyses: default_tile.sex#ANALYSIS_THRESH = 1.0; default_tile.sex#DETECT_MINAREA = 3; default_tile.sex#FILTER = Y; - default_tile.sex#MEMORY_PIXSTACK = 300000; + default_tile.sex#MEMORY_PIXSTACK = 3000000; config_tile_Sx.ini#SEXTRACTOR_RUNNER.DOT_CONV_FILE = $SP_CONFIG/gauss_3.0_7x7.conv; config_exp_psfex.ini#SEXTRACTOR_RUNNER.DOT_SEX_FILE = diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex index dd44248aa..cb6370e4c 100644 --- a/workflow/config/cfis/default_tile.sex +++ b/workflow/config/cfis/default_tile.sex @@ -121,8 +121,10 @@ BACK_FILTTHRESH 0.0 # Threshold above which the background- #--------------------- Memory (change with caution!) ------------------------- MEMORY_OBJSTACK 3000 # number of objects in stack +# Large enough that no object overflows the stack, a bright star's halo +# included (300000 overflows on such tiles and truncates the object). # @sc [decision:detection.detection_threshold_policy] -MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_PIXSTACK 3000000 # number of pixels in stack MEMORY_BUFSIZE 1024 # number of lines in buffer #------------------------------- ASSOCiation --------------------------------- diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index cc70e96b0..107bce4f9 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -542,12 +542,12 @@ rule tile_detect: # MEASURED on eight DR6 tiles chosen to span conditions (high latitude, # b = 18 deg, the A2199 cluster, Alioth's halo, two survey-edge tiles at # 96% zero weight; 1-10 exposures), the step as shapepipe_run runs it on - # candide: wall time 23-143 s (SExtractor + join <= 50 s, post-processing - # <= 92 s), peak RSS 1.75 GiB, set by the join holding the ~430 MB + # candide: wall time 23-160 s (SExtractor + join <= 93 s, post-processing + # <= 97 s), peak RSS 1.84 GiB, set by the join holding the ~430 MB # catalogue twice. Image simulations run the same step on tiles of the same # size (10000 x 10000 px); SExtractor alone on one takes 34 s and 0.74 GiB. - # 4000 MB is 2.2x the worst tile, and an OOM retries at 8000. The runtime - # is ~8x the slowest tile, for /scratch I/O on nibi (a 400 MB image and + # 4000 MB is 2.1x the worst tile, and an OOM retries at 8000. The runtime + # is ~7x the slowest tile, for /scratch I/O on nibi (a 400 MB image and # weight in, a ~430 MB catalogue out). nibi bills max(cores, mem_GB/4), so # the job bills 1 core-equivalent. threads: 1 From 5983208182dce7de42aa216b9be8330ff0d68019 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 05:24:34 +0200 Subject: [PATCH 38/44] match_catalogue: guard at 2% beyond 20 unpaired per side The guard catches other pixels (DR5 pairs ~95%), not deblending near large objects (DR6 tiles pair at least 99.2%) or a few children on a near-empty edge tile. New key MATCH_TOLERATED_UNPAIRED = 20; MATCH_MIN_FRACTION = 0.98. Co-Authored-By: Claude Opus 5.5 --- .../sextractor_package/match_catalogue.py | 47 ++++++++++++------- src/shapepipe/modules/sextractor_runner.py | 3 ++ tests/module/test_match_catalogue.py | 44 ++++++++++++++++- workflow/config/cfis/config_tile_Sx.ini | 15 ++++-- 4 files changed, 84 insertions(+), 25 deletions(-) diff --git a/src/shapepipe/modules/sextractor_package/match_catalogue.py b/src/shapepipe/modules/sextractor_package/match_catalogue.py index a6b70f8d4..68bca78f7 100644 --- a/src/shapepipe/modules/sextractor_package/match_catalogue.py +++ b/src/shapepipe/modules/sextractor_package/match_catalogue.py @@ -92,8 +92,8 @@ def relabel(seg_vignets, old_number, new_number): return table[seg_vignets] -def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, - w_log=None): +def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98, + tolerated_unpaired=20, w_log=None): """Take membership and ``NUMBER`` from an external catalogue. Each SExtractor row is paired with its mutual nearest neighbour in the @@ -105,15 +105,16 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, other HDU and column is kept. Both catalogues come from the same pixels, so pairs agree to ~1e-4 - pixel. On eight DR6 tiles at least 99.6% of SExtractor rows and 99.4% of - UNIONS objects pair; the shortfall is SExtractor deblending differently - near very large objects (a cD galaxy, a bright star's halo). Run on - different pixels (a DR5 image against the DR6 catalogue) only ~95% pair. - So the run stops when fewer than ``min_fraction`` of the SExtractor rows, - or of the external objects, pair: the first catches a catalogue from - other pixels, the second a detection configuration that finds fewer - objects than the catalogue's, which would leave its objects without - shapes. + pixel, and on eight DR6 tiles at least 99.2% of each side pairs; the + shortfall is deblending that differs near very large objects (a cD + galaxy, a bright star's halo). On different pixels (a DR5 image against + the DR6 catalogue) only ~95% pair. The guard is there to catch that: the + run stops when the unpaired rows of either side exceed both + ``(1 - min_fraction)`` of that side and ``tolerated_unpaired``, so a + near-empty edge tile with a couple of unpaired children passes. On the + SExtractor side it catches a catalogue from other pixels; on the + external side, a detection configuration that finds fewer objects than + the catalogue's, which would leave its objects without shapes. Parameters ---------- @@ -126,7 +127,10 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, Largest separation of a pair, in pixels; default 1 min_fraction : float, optional Smallest acceptable fraction of SExtractor rows, and of external - objects, with a partner; default 0.99 + objects, with a partner; default 0.98 + tolerated_unpaired : int, optional + Number of unpaired rows on either side accepted whatever the + fraction; default 20 w_log : logging.Logger, optional Pipeline logger @@ -140,8 +144,8 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, Raises ------ ValueError - If fewer than ``min_fraction`` of the SExtractor rows, or of the - external objects, pair + If more than ``tolerated_unpaired`` rows of either side, and more + than ``1 - min_fraction`` of it, have no partner @sc [decision:detection.tile_detection] """ @@ -162,21 +166,28 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.99, n_external_only=len(ext) - len(i_sex), ) summary = ", ".join(f"{k}={v}" for k, v in counts.items()) + def too_many_unpaired(n_total): + unpaired = n_total - len(i_sex) + return (unpaired > tolerated_unpaired + and unpaired > (1 - min_fraction) * n_total) + fraction = len(i_sex) / max(len(data), 1) - if fraction < min_fraction: + if too_many_unpaired(len(data)): raise ValueError( f"Only {fraction:.4f} of the SExtractor rows of {cat_path} pair" + f" with {ext_cat_path} within {radius} px (minimum" - + f" {min_fraction}; {summary}). The catalogue must come from the" + + f" {min_fraction} beyond {tolerated_unpaired} unpaired;" + + f" {summary}). The catalogue must come from the" + " same pixels as the image: check that the tile image is the" + " release of the catalogue (DR6)." ) fraction_ext = len(i_sex) / max(len(ext), 1) - if fraction_ext < min_fraction: + if too_many_unpaired(len(ext)): raise ValueError( f"Only {fraction_ext:.4f} of the objects of {ext_cat_path} pair" + f" with a row of {cat_path} within {radius} px (minimum" - + f" {min_fraction}; {summary}). SExtractor finds fewer objects" + + f" {min_fraction} beyond {tolerated_unpaired} unpaired;" + + f" {summary}). SExtractor finds fewer objects" + " than the catalogue: check that the detection configuration" + " is the catalogue's." ) diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index 92599be4a..65181712f 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -126,6 +126,9 @@ def sextractor_runner( min_fraction=config.getfloat( module_config_sec, "MATCH_MIN_FRACTION" ), + tolerated_unpaired=config.getint( + module_config_sec, "MATCH_TOLERATED_UNPAIRED" + ), w_log=w_log, ) diff --git a/tests/module/test_match_catalogue.py b/tests/module/test_match_catalogue.py index 2d6dedb21..6b7d47c46 100644 --- a/tests/module/test_match_catalogue.py +++ b/tests/module/test_match_catalogue.py @@ -122,7 +122,7 @@ def test_too_few_catalogue_objects_paired_stop_the_run(tmp_path): cat = _sexcat(tmp_path / "sexcat.fits", X[:4], Y[:4]) ext = _external(tmp_path / "ext.cat", [1, 2, 3, 4, 5], X, Y) with pytest.raises(ValueError, match="fewer objects"): - mc.match_catalogue(cat, ext, min_fraction=0.99) + mc.match_catalogue(cat, ext, min_fraction=0.99, tolerated_unpaired=0) assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4] counts = mc.match_catalogue(cat, ext, min_fraction=0.8) assert counts["n_paired"] == 4 and counts["n_external_only"] == 1 @@ -133,13 +133,53 @@ def test_too_few_pairs_stop_the_run(tmp_path): cat = _sexcat(tmp_path / "sexcat.fits", X, Y) ext = _external(tmp_path / "ext.cat", [1, 2, 3, 4], X[:4], Y[:4]) with pytest.raises(ValueError, match="DR6"): - mc.match_catalogue(cat, ext, min_fraction=0.99) + mc.match_catalogue(cat, ext, min_fraction=0.99, tolerated_unpaired=0) # Nothing was written. assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4, 5] mc.match_catalogue(cat, ext, min_fraction=0.8) assert list(_objects(cat)["NUMBER"]) == [1, 2, 3, 4] +def _grid(n): + """n well-separated positions, 10 px apart.""" + i = np.arange(n) + return 5.0 + 10.0 * (i % 50), 5.0 + 10.0 * (i // 50) + + +def test_small_tile_with_few_unpaired_children_passes(tmp_path): + """An edge tile: 5 of 100 catalogue objects unpaired (5%) is within the + 20 tolerated, so the run goes on with the default guard.""" + x, y = _grid(105) + cat = _sexcat(tmp_path / "sexcat.fits", x[:100], y[:100]) + ext = _external(tmp_path / "ext.cat", np.arange(1, 106), x, y) + counts = mc.match_catalogue(cat, ext) + assert counts["n_paired"] == 100 and counts["n_external_only"] == 5 + + +def test_large_tile_with_unpaired_children_passes(tmp_path): + """A cluster tile: 30 of 2000 catalogue objects unpaired (1.5%) is more + than 20 but under 2%, so the run goes on with the default guard.""" + x, y = _grid(2000) + cat = _sexcat(tmp_path / "sexcat.fits", x[:1970], y[:1970]) + ext = _external(tmp_path / "ext.cat", np.arange(1, 2001), x, y) + counts = mc.match_catalogue(cat, ext) + assert counts["n_external_only"] == 30 + + +@pytest.mark.parametrize("side", ["sextractor", "external"]) +def test_other_pixels_stop_the_run(tmp_path, side): + """5% unpaired on either side (a DR5 image: ~95% pair) stops the run with + the default guard, far beyond the 20 tolerated.""" + x, y = _grid(2000) + n_sex, n_ext = (2000, 1900) if side == "sextractor" else (1900, 2000) + cat = _sexcat(tmp_path / "sexcat.fits", x[:n_sex], y[:n_sex]) + ext = _external(tmp_path / "ext.cat", np.arange(1, n_ext + 1), + x[:n_ext], y[:n_ext]) + with pytest.raises(ValueError, match="DR6" if side == "sextractor" + else "fewer objects"): + mc.match_catalogue(cat, ext) + + def test_seg_vignet_is_relabelled(tmp_path): """SEG_VIGNET labels follow NUMBER; a dropped row's footprint is -1.""" x, y = X[:3], Y[:3] diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini index 524635054..43a59c493 100644 --- a/workflow/config/cfis/config_tile_Sx.ini +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -131,12 +131,17 @@ MATCH_CATALOGUE = ${SP_MATCH_CATALOGUE:-} # @sc [decision:detection.tile_detection] MATCH_RADIUS = 1.0 -# Smallest fraction of SExtractor rows, and of catalogue objects, that must -# pair; fewer stops the run. DR6 tiles pair at least 99.6% of rows and 99.4% -# of objects (the lowest near a cD galaxy or a bright star); a catalogue on other -# pixels (the DR5 image of a tile against its DR6 catalogue) pairs ~95%. +# The run stops when the unpaired rows of either side (SExtractor rows, +# catalogue objects) exceed both (1 - MATCH_MIN_FRACTION) of that side and +# MATCH_TOLERATED_UNPAIRED. The guard catches a catalogue measured on other +# pixels (the DR5 image of a tile against its DR6 catalogue pairs ~95%), not +# deblending near very large objects: DR6 tiles pair at least 99.2% of each +# side, and a near-empty edge tile may leave a few children unpaired. # @sc [decision:detection.tile_detection] -MATCH_MIN_FRACTION = 0.99 +MATCH_MIN_FRACTION = 0.98 + +# @sc [decision:detection.tile_detection] +MATCH_TOLERATED_UNPAIRED = 20 ## Post-processing From 2a058c06c1f841f88a00be8f9cbbea614ea9583c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 18:49:45 -0400 Subject: [PATCH 39/44] SEG_VIGNET keeps own-vs-other through the DR6 join match_catalogue relabels every SEG_VIGNET label through the whole SExtractor -> DR6 map, unpaired footprints to UNMATCHED_LABEL (-1). That separates each row's own footprint from every other one only if the catalogue's NUMBERs are unique and positive, so the join now refuses a catalogue where they are not. Tests, through sextractor_runner with SExtractor's outputs stubbed: on a crowd where each paired row's DR6 NUMBER is another object's SExtractor NUMBER (one of them the dropped detection's), each own footprint carries the final NUMBER and nothing else does, the dropped footprint is -1, and uberseg_weight and seg_has_neighbour see the stamps as they saw the SExtractor-labelled ones. Sims (no join) keep the check image's stamps. A negative control shows a partial relabel collides. make_cat's exclusion of SEG_VIGNET moves here from the deleted converter tests. Swapping the join before add_seg_vignet, relabelling only paired labels, or dropping the NUMBER guard each fails a test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017JQtu8oaZQEAqzbxZbPpWs --- .../sextractor_package/match_catalogue.py | 25 +- tests/module/test_sextractor_seg_vignet.py | 234 ++++++++++++++++++ 2 files changed, 252 insertions(+), 7 deletions(-) diff --git a/src/shapepipe/modules/sextractor_package/match_catalogue.py b/src/shapepipe/modules/sextractor_package/match_catalogue.py index 68bca78f7..da537026f 100644 --- a/src/shapepipe/modules/sextractor_package/match_catalogue.py +++ b/src/shapepipe/modules/sextractor_package/match_catalogue.py @@ -23,7 +23,8 @@ # The SEG_VIGNET label of a footprint whose SExtractor row has no partner in # the external catalogue, and so leaves the catalogue. Negative, so it never -# collides with a NUMBER; UberSeg only asks "self or not self". +# collides with a NUMBER (match_catalogue requires positive ones); UberSeg only +# asks "self or not self". UNMATCHED_LABEL = -1 @@ -100,9 +101,11 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98, external catalogue (``X_IMAGE``, ``Y_IMAGE``, same image grid) within ``radius`` pixels. Paired rows take the external ``NUMBER``; unpaired rows leave the catalogue. A ``SEG_VIGNET`` column, when present, is - relabelled to the new numbering, with the footprints of rows that left - marked ``UNMATCHED_LABEL``. The catalogue is rewritten in place; every - other HDU and column is kept. + relabelled through the whole map from old to new numbers, with the + footprints of rows that left marked ``UNMATCHED_LABEL``: as the external + numbers are unique and positive, each row's own footprint carries its new + ``NUMBER`` and no other footprint can, whatever the two numberings share. + The catalogue is rewritten in place; every other HDU and column is kept. Both catalogues come from the same pixels, so pairs agree to ~1e-4 pixel, and on eight DR6 tiles at least 99.2% of each side pairs; the @@ -144,13 +147,21 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98, Raises ------ ValueError - If more than ``tolerated_unpaired`` rows of either side, and more - than ``1 - min_fraction`` of it, have no partner + If the external ``NUMBER`` repeats or is not positive, or if more + than ``tolerated_unpaired`` rows of either side, and more than + ``1 - min_fraction`` of it, have no partner @sc [decision:detection.tile_detection] """ ext = asc.read(ext_cat_path, format="sextractor", include_names=["NUMBER", "X_IMAGE", "Y_IMAGE"]) + ext_number = np.asarray(ext["NUMBER"]) + if (ext_number <= 0).any() or len(np.unique(ext_number)) < len(ext): + raise ValueError( + f"{ext_cat_path} has a repeated or non-positive NUMBER; the join" + + " needs unique positive numbers, so that no relabelled" + + " SEG_VIGNET footprint takes another object's number." + ) with fits.open(cat_path) as hdul: hdus = [hdu.copy() for hdu in hdul] objects = next(h for h in hdus if h.name == "LDAC_OBJECTS") @@ -194,7 +205,7 @@ def too_many_unpaired(n_total): old_number = np.asarray(data["NUMBER"]) new_number = np.full(len(data), UNMATCHED_LABEL, np.int64) - new_number[i_sex] = np.asarray(ext["NUMBER"])[i_ext] + new_number[i_sex] = ext_number[i_ext] columns = [] for col in objects.columns: diff --git a/tests/module/test_sextractor_seg_vignet.py b/tests/module/test_sextractor_seg_vignet.py index 5f5eb13d0..3872baeac 100644 --- a/tests/module/test_sextractor_seg_vignet.py +++ b/tests/module/test_sextractor_seg_vignet.py @@ -167,3 +167,237 @@ def test_sextractor_caller_names_its_check_images(tmp_path): "BACKGROUND": f"{tmp_path}/background-001-001.fits", "SEGMENTATION": f"{tmp_path}/segmentation-001-001.fits", } + + +# --- SEG_VIGNET through the join to the UNIONS catalogue ------------------- +# +# UberSeg (uberseg_weight, seg_has_neighbour, Ngmix._check_central_seg_label) +# compares each stamp's labels with the row's NUMBER. The check image is +# labelled with SExtractor's NUMBERs; the join replaces NUMBER with the +# catalogue's, and relabels every stamp through the whole SExtractor -> UNIONS +# map, the footprints of rows it drops included (UNMATCHED_LABEL). The scene +# makes old and new numbers collide: each paired row's new NUMBER is another +# object's SExtractor NUMBER, one of them the dropped detection's. + +CROWD_STAMP = 9 +# (SExtractor NUMBER, x, y, UNIONS NUMBER or None for no partner) +CROWD = [ + (1, 10.0, 10.0, 2), + (2, 13.0, 10.0, None), + (3, 10.0, 13.0, 1), + (4, 22.0, 22.0, 3), +] +EXT_ONLY = (7, 5.0, 25.0) +EXT_HEADER = ( + "# 1 NUMBER Running object number\n" + "# 2 X_IMAGE Object position along x [pixel]\n" + "# 3 Y_IMAGE Object position along y [pixel]\n" +) + + +def _crowd_seg(): + seg = np.zeros((30, 30), np.int32) + for number, x, y, _ in CROWD: + col, row = int(x) - 1, int(y) - 1 + seg[row - 1:row + 2, col - 1:col + 2] = number + return seg + + +def _write_crowd(sexcat_path, seg_path): + """SExtractor's outputs for CROWD: the sexcat (with the double positions + seg_vignet_param_file asks for) and the SEGMENTATION check image.""" + number = np.array([o[0] for o in CROWD], np.int32) + x = np.array([o[1] for o in CROWD]) + y = np.array([o[2] for o in CROWD]) + n = len(number) + size = CROWD_STAMP * CROWD_STAMP + objects = fits.BinTableHDU.from_columns([ + fits.Column(name="NUMBER", format="J", array=number), + fits.Column(name="X_IMAGE", format="E", array=x.astype(np.float32)), + fits.Column(name="Y_IMAGE", format="E", array=y.astype(np.float32)), + fits.Column(name="VIGNET", format=f"{size}E", + array=np.ones((n, size), np.float32), + dim=f"({CROWD_STAMP},{CROWD_STAMP})"), + fits.Column(name="X_IMAGE_DBL", format="D", array=x), + fits.Column(name="Y_IMAGE_DBL", format="D", array=y), + ], name="LDAC_OBJECTS") + imhead = fits.BinTableHDU.from_columns( + [fits.Column(name="Field Header Card", format="80A", + array=np.array(["HISTORY x"]))], name="LDAC_IMHEAD") + fits.HDUList([fits.PrimaryHDU(), imhead, objects]).writeto(sexcat_path) + fits.PrimaryHDU(_crowd_seg()).writeto(seg_path) + + +def _write_crowd_external(path): + rows = [(new, x, y) for _, x, y, new in CROWD if new is not None] + rows.append(EXT_ONLY) + path.write_text(EXT_HEADER + "".join( + f"{n:10d} {x:11.4f} {y:11.4f}\n" for n, x, y in rows)) + return str(path) + + +def _old_stamps(): + """The check image's stamps in SExtractor's numbering, row by row.""" + col = np.array([int(o[1]) - 1 for o in CROWD]) + row = np.array([int(o[2]) - 1 for o in CROWD]) + return ss.cut_stamps(_crowd_seg(), col, row, CROWD_STAMP, 0) + + +def _run_runner(tmp_path, monkeypatch, match): + """sextractor_runner as the workflow runs it under blend_handling: + uberseg, with SExtractor replaced by the CROWD outputs.""" + import logging + + from shapepipe.modules import sextractor_runner as runner_module + from shapepipe.pipeline.config import CustomParser + + out = tmp_path / "output" + tmp = tmp_path / "tmp" + out.mkdir() + tmp.mkdir() + num = "-001-001" + + def fake_execute(command_line): + _write_crowd(out / f"sexcat{num}.fits", out / f"segmentation{num}.fits") + return "", "All done" + + monkeypatch.setattr(runner_module, "execute", fake_execute) + dot_param = tmp_path / "default.param" + dot_param.write_text("NUMBER\nX_IMAGE\nY_IMAGE\nVIGNET(9,9)\n") + ext = _write_crowd_external(tmp_path / "CFIS_cat-001-001.cat") + config = CustomParser() + config["SEXTRACTOR_RUNNER"] = { + "EXEC_PATH": "source-extractor", + "DOT_SEX_FILE": "d.sex", "DOT_PARAM_FILE": str(dot_param), + "DOT_CONV_FILE": "d.conv", "WEIGHT_IMAGE": "True", + "FLAG_IMAGE": "False", "PSF_FILE": "False", + "DETECTION_IMAGE": "False", "DETECTION_WEIGHT": "False", + "ZP_FROM_HEADER": "False", "BKG_FROM_HEADER": "False", + "CHECKIMAGE": "BACKGROUND, SEGMENTATION", + "SEG_VIGNET": "${SP_SEG_VIGNET:-False}", + "MATCH_CATALOGUE": "${SP_MATCH_CATALOGUE:-}", + "MATCH_RADIUS": "1.0", "MATCH_MIN_FRACTION": "0.98", + "MATCH_TOLERATED_UNPAIRED": "20", "MAKE_POST_PROCESS": "False", + } + monkeypatch.setenv("SP_SEG_VIGNET", "True") + monkeypatch.setenv("SP_MATCH_CATALOGUE", ext if match else "") + runner_module.sextractor_runner( + [str(tmp_path / f"image{num}.fits"), + str(tmp_path / f"weight{num}.fits")], + {"output": str(out), "tmp": str(tmp)}, num, config, + "SEXTRACTOR_RUNNER", logging.getLogger("test"), + ) + with fits.open(out / f"sexcat{num}.fits") as hdul: + return hdul["LDAC_OBJECTS"].data.copy() + + +def test_runner_seg_vignet_carries_the_final_number(tmp_path, monkeypatch): + """Through the runner, each row's own footprint carries its final + (UNIONS) NUMBER and nothing else does, though every new NUMBER is some + other object's SExtractor NUMBER; the dropped detection's footprint is + UNMATCHED_LABEL. Fails if the join runs before add_seg_vignet, or relabels + only the paired labels.""" + from shapepipe.modules.sextractor_package import match_catalogue as mc + + data = _run_runner(tmp_path, monkeypatch, match=True) + paired = [i for i, o in enumerate(CROWD) if o[3] is not None] + old = _old_stamps()[paired] + old_number = np.array([CROWD[i][0] for i in paired]) + new_number = np.array([CROWD[i][3] for i in paired]) + npt.assert_array_equal(data["NUMBER"], new_number) + seg = data["SEG_VIGNET"] + assert "X_IMAGE_DBL" not in data.names + centre = CROWD_STAMP // 2 + npt.assert_array_equal(seg[:, centre, centre], data["NUMBER"]) + dropped = CROWD[1][0] + for i in range(len(paired)): + npt.assert_array_equal(seg[i] == new_number[i], + old[i] == old_number[i]) + npt.assert_array_equal(seg[i] == mc.UNMATCHED_LABEL, + old[i] == dropped) + npt.assert_array_equal(seg[i] == 0, old[i] == 0) + # Rows 0 and 1 overlap the dropped detection, whose SExtractor label (2) + # is row 0's new NUMBER. + assert (seg[0] == mc.UNMATCHED_LABEL).any() + assert (old[0] == new_number[0]).any() + + +def test_runner_seg_vignet_without_a_join_keeps_sextractor_numbers( + tmp_path, monkeypatch, +): + """Image simulations (empty MATCH_CATALOGUE): every row stays, and the + stamps are the check image's, labelled with SExtractor's NUMBER.""" + data = _run_runner(tmp_path, monkeypatch, match=False) + npt.assert_array_equal(data["NUMBER"], [o[0] for o in CROWD]) + npt.assert_array_equal(data["SEG_VIGNET"], _old_stamps()) + + +def test_uberseg_sees_the_same_neighbours_after_the_join(tmp_path, + monkeypatch): + """UberSeg only asks own-versus-other, so the relabelled stamps give the + weights and the neighbour flag the SExtractor-labelled ones give.""" + from shapepipe.modules.ngmix_package.ngmix import ( + seg_has_neighbour, + uberseg_weight, + ) + + data = _run_runner(tmp_path, monkeypatch, match=True) + paired = [i for i, o in enumerate(CROWD) if o[3] is not None] + old = _old_stamps()[paired] + weight = np.ones((CROWD_STAMP, CROWD_STAMP)) + for i, j in enumerate(paired): + new_seg, number = data["SEG_VIGNET"][i], data["NUMBER"][i] + npt.assert_array_equal( + uberseg_weight(weight, new_seg, number), + uberseg_weight(weight, old[i], CROWD[j][0])) + assert (seg_has_neighbour(new_seg, number) + == seg_has_neighbour(old[i], CROWD[j][0])) + assert seg_has_neighbour(data["SEG_VIGNET"][0], data["NUMBER"][0]) + assert not seg_has_neighbour(data["SEG_VIGNET"][2], data["NUMBER"][2]) + + +def test_a_partial_relabel_would_collide(): + """The negative control: relabelling only the paired labels leaves the + dropped detection's SExtractor label 2 in row 0's stamp, equal to row 0's + new NUMBER, so UberSeg would take the neighbour for the object.""" + old = _old_stamps()[0] + partial = old.copy() + for number, _, _, new in CROWD: + if new is not None: + partial[old == number] = new + assert ((partial == 2) & (old != 1)).any() + + +def test_join_rejects_a_catalogue_with_repeated_or_nonpositive_numbers( + tmp_path, +): + """Unique positive catalogue NUMBERs are what keep a relabelled stamp's + own footprint apart from every other one.""" + from shapepipe.modules.sextractor_package import match_catalogue as mc + + for numbers in ([5, 5, 6], [0, 5, 6]): + case = tmp_path / str(numbers[0] + numbers[1]) + case.mkdir() + _write_crowd(case / "sexcat.fits", case / "seg.fits") + rows = zip(numbers, [10.0, 10.0, 22.0], [10.0, 13.0, 22.0]) + ext = case / "ext.cat" + ext.write_text(EXT_HEADER + "".join( + f"{n:10d} {x:11.4f} {y:11.4f}\n" for n, x, y in rows)) + with pytest.raises(ValueError, match="NUMBER"): + mc.match_catalogue(str(case / "sexcat.fits"), str(ext)) + + +def test_seg_vignet_stays_out_of_the_final_catalogue(tmp_path): + """make_cat drops SEG_VIGNET with VIGNET: the stamps are ngmix inputs, + not catalogue columns.""" + from shapepipe.modules.make_cat_package import make_cat + + _write_crowd(tmp_path / "sexcat-001-001.fits", tmp_path / "seg.fits") + ss.add_seg_vignet(str(tmp_path / "sexcat-001-001.fits"), + str(tmp_path / "seg.fits")) + final = make_cat.prepare_final_cat_file(str(tmp_path), "-001-001") + make_cat.save_sextractor_data(final, str(tmp_path / "sexcat-001-001.fits")) + with fits.open(tmp_path / "final_cat-001-001.fits") as hdul: + names = hdul["RESULTS"].data.names + assert "VIGNET" not in names and "SEG_VIGNET" not in names + assert "NUMBER" in names From 45eef5d4afbccdd6b26509976eb288d373f50790 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 19:00:04 -0400 Subject: [PATCH 40/44] tile_detect: 3000 MB more under uberseg for SEG_VIGNET Measured on DR6 tile 202.301 with sextractor_runner as tile_detect runs it (post-processing off): peak RSS 1.53 GiB under noisefill, 2.93 GiB under uberseg. add_seg_vignet reads the 400 MB check image and adds an int32 column the size of VIGNET, which the join holds twice. Scaled to #933's 1.84 GiB worst tile, uberseg needs ~3.5 GiB, above the 4000 MB noisefill request; 7000 MB is 2x that. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017JQtu8oaZQEAqzbxZbPpWs --- tests/workflow/README.md | 2 +- tests/workflow/test_dag.py | 6 ++++-- workflow/rules/tile.smk | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/workflow/README.md b/tests/workflow/README.md index 1c29af03c..9748e9126 100644 --- a/tests/workflow/README.md +++ b/tests/workflow/README.md @@ -76,7 +76,7 @@ Apply mutations only to a disposable checkout, run the named test without `--upd | `test_clean_exposure_waits_on_persist_iff_psf` | Drop the persist edge; make it unconditional under fake PSFs; drop vignets consumers; remove the in-scope consumer filter. | | `test_final_cat_merge_reads_every_ready_tile` | Drop one ready tile; append an out-of-scope tile. | | `test_products_use_products_dir_and_run_name` | Rename either merged catalogue or the persist manifest; route products to scratch; derive `CAMPAIGN` from the products directory's basename. | -| `test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg` | Make the Snakefile or config.yaml default `noisefill`; export `BLEND_ENV` under noisefill; drop `blend_env(...)` from either tile_detect or tile_ngmix; export the wrong value; set `NGMIX_SEG_MEM_MB` to 0; give a committed ini's `SEG_VIGNET` / `BLEND_HANDLING` a literal or the wrong default. | +| `test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg` | Make the Snakefile or config.yaml default `noisefill`; export `BLEND_ENV` under noisefill; drop `blend_env(...)` from either tile_detect or tile_ngmix; export the wrong value; set `NGMIX_SEG_MEM_MB` or `DETECT_SEG_MEM_MB` to 0; give a committed ini's `SEG_VIGNET` / `BLEND_HANDLING` a literal or the wrong default. | | `test_unknown_blend_handling_fails_during_parse` | Remove the Snakefile's `blend_handling` check. | | `test_missing_run_fails_during_parse` | Remove `run` from `run_config.REQUIRED`; literal paths must still receive the required-key diagnostic, not a later `KeyError`. | | `test_unit_pre_changes_at_campaign_boundary` | Append a line to `unit_pre`; change one rule's `params.pre`; change one shell; change a rendered thread count. | diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index 8d7787d3c..6a8ccafd3 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -142,6 +142,8 @@ def test_mccd_is_refused_during_parse(tmp_path, resolve_dag): # --- blend_handling --------------------------------------------------------- +# The memory each stage adds under uberseg for SEG_VIGNET. +SEG_MEM_MB = {"tile_detect": 3000, "tile_ngmix": 500} BLEND_EXPORTS = {"tile_detect": {"SP_SEG_VIGNET": "True"}, "tile_ngmix": {"SP_BLEND_HANDLING": "uberseg"}} # The option each stage's committed ini reads the export through, and the @@ -183,7 +185,7 @@ def test_blend_handling_reaches_detection_and_ngmix_only_under_uberseg( tmp_path, resolve_dag, monkeypatch, detection): """A campaign without the knob plans exactly the uberseg campaign. Against explicit noisefill, uberseg only adds its two exports to tile_detect's and - tile_ngmix's prologues and the seg stamps' memory to tile_ngmix, and each + tile_ngmix's prologues and the seg stamps' memory to both, and each export turns its committed ini's option from the noise-fill default to the uberseg value.""" surfaces = {} @@ -234,7 +236,7 @@ def without_exports(text): monkeypatch) == default assert _committed_value(shell, config_dir, option, exports, monkeypatch) == value - assert mem == (nf_mem + 500 if rule == "tile_ngmix" else nf_mem), key + assert mem == nf_mem + SEG_MEM_MB.get(rule, 0), key def test_unknown_blend_handling_fails_during_parse(tmp_path, resolve_dag): diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index a9c5fbccd..1919fa6d5 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -554,6 +554,16 @@ def detect_env(tile): return {"SP_MATCH_CATALOGUE": match, **blend_env("tile_detect")} +# tile_detect's extra memory under uberseg: add_seg_vignet reads the 400 MB +# SEGMENTATION check image and adds an int32 SEG_VIGNET column the size of +# VIGNET, which the join then holds twice. MEASURED on DR6 tile 202.301 +# (36,022 detections; sextractor_runner as tile_detect runs it, without the +# post-processing): peak RSS 1.53 GiB under noisefill, 2.93 GiB under uberseg. +# Scaled to the 1.84 GiB worst tile above, uberseg needs ~3.5 GiB; 7000 MB is +# 2x that. +DETECT_SEG_MEM_MB = 3000 if BLEND_HANDLING == "uberseg" else 0 + + # SExtractor object detection on the tile; under unions_catalogue joined to # the UNIONS catalogue (config_tile_Sx.ini, MATCH_CATALOGUE). rule tile_detect: @@ -583,11 +593,11 @@ rule tile_detect: # 4000 MB is 2.1x the worst tile, and an OOM retries at 8000. The runtime # is ~7x the slowest tile, for /scratch I/O on nibi (a 400 MB image and # weight in, a ~430 MB catalogue out). nibi bills max(cores, mem_GB/4), so - # the job bills 1 core-equivalent. + # the job bills 1 core-equivalent; under uberseg, DETECT_SEG_MEM_MB below. threads: 1 retries: 1 resources: - mem_mb = lambda wc, attempt: 4000 * attempt, + mem_mb = lambda wc, attempt: (4000 + DETECT_SEG_MEM_MB) * attempt, runtime = lambda wc, attempt: 20 * attempt shell: sp_shell("tile_detect", "config_tile_Sx.ini") From ed6a3dbc6cff6dd3be4582b9d926c930a4b86850 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 19:00:05 -0400 Subject: [PATCH 41/44] Update the params pins for UberSeg on #933's detection Regenerated with --update-params-pin in shapepipe-develop-240b37e4.sif with host snakemake 9.23.1. The noisefill pin is #933's params_pin.json byte for byte, so blend_handling: noisefill plans exactly #933's chain; the uberseg default moves in tile_detect and tile_ngmix only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017JQtu8oaZQEAqzbxZbPpWs --- tests/workflow/params_pin.json | 6 +++--- tests/workflow/params_pin_noisefill.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index 6de487277..1744bb06d 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -11,7 +11,7 @@ "final_cat_merge": "e7f46859c4503a2220713d7bb2507555515d0a9632d780b20f14c59e32210023", "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", - "tile_detect": "8336b148769e9d43b64f0d945c05e7d163c64dba4e6287e8c1964d1502e3ffe1", + "tile_detect": "24d015063fb07ba4a48b961284fa1345675f13e167acc62a2c7b397795113b82", "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", @@ -19,12 +19,12 @@ "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23", "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb", "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111", - "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415", + "tile_ngmix": "5b86694f3bac7b8c3f62d6c1df72b38a8b9173fb69ab53c75d7d70e38f5dc9ff", "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3", "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "d67ee5da728e8fee513e1f01a522b386eb6a86d21c9a36250afd6013a06301d5", + "sha256": "7d03fd3acb7608ed1eda69b162f46b2c88eddb8bfa441cbf66245a65510d809b", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", diff --git a/tests/workflow/params_pin_noisefill.json b/tests/workflow/params_pin_noisefill.json index eeb61d950..6de487277 100644 --- a/tests/workflow/params_pin_noisefill.json +++ b/tests/workflow/params_pin_noisefill.json @@ -8,10 +8,10 @@ "exp_persist": "302e2837542bc1102430c27c81c600b7cda32e8bddcb5fd60d33950987609fff", "exp_psf": "2c4f6d00f1939ccbf05b4982a0202a0ff92a4373a727f4e00f3aaab4eba03352", "exp_split": "6e954f8f3d06f44d3f164675912ce27d6216648855d04968f9168bd7d0f2c4fa", - "final_cat_merge": "f6222c98be9777cbab4131d8857bd3817719df9302d0a321c6cd99f67d5bdc76", + "final_cat_merge": "e7f46859c4503a2220713d7bb2507555515d0a9632d780b20f14c59e32210023", "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", - "tile_detect": "1b4b0871bf28296f8e5d6855ae47ff2cb84542c93593a5ae5c0ffc3c84c45479", + "tile_detect": "8336b148769e9d43b64f0d945c05e7d163c64dba4e6287e8c1964d1502e3ffe1", "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", @@ -24,7 +24,7 @@ "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e", + "sha256": "d67ee5da728e8fee513e1f01a522b386eb6a86d21c9a36250afd6013a06301d5", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", From b152c762fa2025aa68751a1ce8584bc60811d261 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 21:19:44 -0400 Subject: [PATCH 42/44] tile.smk: keep the catalogue path on one line, as the tile-detection test reads it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017JQtu8oaZQEAqzbxZbPpWs --- workflow/rules/tile.smk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 1919fa6d5..8dc767443 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -548,8 +548,7 @@ def detect_env(tile): """ match = "" if TILE_DETECTION == "unions_catalogue": - gic = (f"{tile_dir(tile)}/output/run_sp_tile_Gic/get_images_runner" - + "/output") + gic = f"{tile_dir(tile)}/output/run_sp_tile_Gic/get_images_runner/output" match = f"{gic}/CFIS_cat{unit_num(tile)}.cat" return {"SP_MATCH_CATALOGUE": match, **blend_env("tile_detect")} From 55af8dce002fbe9bc03aa9cfee866cd93ba00188 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 21:38:04 -0400 Subject: [PATCH 43/44] SEG_VIGNET centres as SExtractor centres VIGNET: (int)(mx + 0.49999) SExtractor 2.25.0 (src/analyse.c, ix=(int)(obj->mx+0.49999); the SIF's Debian 2.25.0+ds-3 does not patch it) truncates the 0-based barycentre plus 0.49999. np.rint differs at exact halves with an odd floor and for fractional parts in (0.5, 0.50001); vignet_centre now mirrors SExtractor. The tests' expected centres are written out by hand from that rule and their stamps cut by plain slicing, not by cut_stamps; they cover exact halves and fractions either side of 0.50001, and an object only the double positions resolve. Reverting to np.rint fails five of them. On DR6 tile 202.301 no object falls where the two rules disagree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017JQtu8oaZQEAqzbxZbPpWs --- .../sextractor_package/sextractor_script.py | 41 +++++++-- tests/module/test_sextractor_seg_vignet.py | 92 ++++++++++++------- 2 files changed, 94 insertions(+), 39 deletions(-) diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index 1c8ebb1cf..44a9eaba9 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -85,13 +85,38 @@ def seg_vignet_column(seg_vignets): DOUBLE_POSITIONS = ("X_IMAGE_DBL", "Y_IMAGE_DBL") +def vignet_centre(pos): + """The 0-based pixel SExtractor centres VIGNET on, along one axis. + + SExtractor 2.25.0 (``src/analyse.c``, ``ix=(int)(obj->mx+0.49999)``, + unchanged in Debian's 2.25.0+ds-3) truncates the 0-based barycentre plus + 0.49999. That is not round-to-nearest: a fractional part in + [0.5, 0.50001) goes down, where ``np.rint`` would go up (or to even). + + Parameters + ---------- + pos : array_like of float + 1-based double-precision position (``X_IMAGE_DBL`` or + ``Y_IMAGE_DBL``; SExtractor writes ``mx + 1``) + + Returns + ------- + numpy.ndarray + 0-based pixel index, int64 + + """ + mx = np.asarray(pos, np.float64) - 1.0 + return np.trunc(mx + 0.49999).astype(np.int64) + + def seg_vignet_param_file(dot_param, output_path): """Write a SExtractor parameter file that also asks for the double positions. - SExtractor centres VIGNET on the pixel nearest its double-precision - barycentre, which the float32 ``X_IMAGE`` / ``Y_IMAGE`` hide on exact - half pixels; ``X_IMAGE_DBL`` / ``Y_IMAGE_DBL`` carry it. + SExtractor centres VIGNET from its double-precision barycentre + (:func:`vignet_centre`), which the float32 ``X_IMAGE`` / ``Y_IMAGE`` + cannot resolve near half pixels; ``X_IMAGE_DBL`` / ``Y_IMAGE_DBL`` + carry it. :func:`add_seg_vignet` reads them and drops them again. Parameters @@ -120,9 +145,9 @@ def add_seg_vignet(cat_path, seg_path, w_log=None): """Add the ``SEG_VIGNET`` column to a SExtractor catalogue. The SEGMENTATION check image, whose labels are the catalogue's - ``NUMBER``, is cut on the grid of each object's VIGNET, centred on the - pixel nearest ``X_IMAGE_DBL`` / ``Y_IMAGE_DBL`` as SExtractor centres - VIGNET (:func:`cut_stamps`), and written, int32 and 0 off the image, as + ``NUMBER``, is cut on the grid of each object's VIGNET, centred where + SExtractor centres VIGNET (:func:`vignet_centre` of ``X_IMAGE_DBL`` / + ``Y_IMAGE_DBL``, :func:`cut_stamps`), and written, int32 and 0 off the image, as ``SEG_VIGNET`` in ``LDAC_OBJECTS``, which ngmix's UberSeg blend handling reads. The double positions (:func:`seg_vignet_param_file`) are dropped; every other HDU and column is kept. @@ -152,8 +177,8 @@ def add_seg_vignet(cat_path, seg_path, w_log=None): f"{cat_path} lacks {', '.join(missing)}, which SEG_VIGNET is" + " centred on; run SExtractor with seg_vignet_param_file." ) - col = np.rint(np.asarray(data["X_IMAGE_DBL"]) - 1).astype(np.int64) - row = np.rint(np.asarray(data["Y_IMAGE_DBL"]) - 1).astype(np.int64) + col = vignet_centre(data["X_IMAGE_DBL"]) + row = vignet_centre(data["Y_IMAGE_DBL"]) size = data["VIGNET"].shape[1] seg_vignets = cut_stamps(fits.getdata(seg_path), col, row, size, 0) kept = [c for c in objects.columns.columns diff --git a/tests/module/test_sextractor_seg_vignet.py b/tests/module/test_sextractor_seg_vignet.py index 3872baeac..f93e1a04a 100644 --- a/tests/module/test_sextractor_seg_vignet.py +++ b/tests/module/test_sextractor_seg_vignet.py @@ -2,11 +2,15 @@ ``sextractor_script.add_seg_vignet`` cuts the SEGMENTATION check image on the grid SExtractor cut each object's VIGNET on and adds it to the sexcat as the -int32 ``SEG_VIGNET`` column. SExtractor centres VIGNET on the pixel nearest -its double-precision barycentre. The catalogue's float32 X_IMAGE / Y_IMAGE -tie a few positions to an exact half pixel, so the runner asks SExtractor -for X_IMAGE_DBL / Y_IMAGE_DBL too (``seg_vignet_param_file``), and -``add_seg_vignet`` centres on them and then drops them. +int32 ``SEG_VIGNET`` column. SExtractor 2.25.0 centres VIGNET on +``(int)(mx + 0.49999)`` of its 0-based double-precision barycentre +(``src/analyse.c``), which the catalogue's float32 X_IMAGE / Y_IMAGE cannot +resolve near half pixels, so the runner asks SExtractor for X_IMAGE_DBL / +Y_IMAGE_DBL too (``seg_vignet_param_file``), and ``add_seg_vignet`` centres +on them and then drops them. + +The expected centres below are written out by hand from that rule, and the +expected stamps are cut by plain slicing, not by the module's helpers. """ import numpy as np @@ -20,22 +24,45 @@ STAMP = 7 BIG = np.float32(-1e30) -# (NUMBER, x, y): 1-based double-precision barycentres. Object 3's x and -# object 4's y sit a float32 ulp either side of a half pixel, so the float32 -# catalogue shows an exact .5 for both but SExtractor rounded them apart, and -# each the other way from rounding the float32 value half to even. +# (NUMBER, X_IMAGE_DBL, Y_IMAGE_DBL, VIGNET centre col, row (0-based)). +# Object 3's x is an exact half with an odd floor (np.rint goes to 32 - 1), +# object 4's y has a fractional part in [0.5, 0.50001) (np.rint goes up), and +# object 6's x is just past 0.50001, where float32 falls back below it. OBJECTS = [ - (1, 20.2, 15.7), - (2, 1.3, 48.9), - (3, 31.5 + 5e-7, 30.1), - (4, 44.8, 22.5 - 5e-7), - (5, 59.9, 1.2), + (1, 20.2, 15.7, 19, 15), + (2, 1.3, 48.9, 0, 48), + (3, 32.5, 30.1, 31, 29), + (4, 44.8, 22.5000099, 44, 21), + (5, 59.9, 1.2, 59, 0), + (6, 31.5000101, 10.2, 31, 9), ] DBL = ["X_IMAGE_DBL", "Y_IMAGE_DBL"] +@pytest.mark.parametrize("pos, centre", [ + (32.5, 31), # exact half, odd floor: down, not to even + (31.5, 30), # exact half, even floor + (31.49999, 30), + (31.5000099, 30), # fraction 0.5000099 < 0.50001: down + (31.5000101, 31), # fraction 0.5000101 > 0.50001: up + (31.4999, 30), + (20.2, 19), + (20.8, 20), + (0.6, 0), # mx = -0.4 truncates to 0 +]) +def test_vignet_centre_is_sextractors_rule(pos, centre): + assert ss.vignet_centre([pos]).tolist() == [centre] + + +def _cut(array, col, row, fill): + """The STAMP stamp of ``array`` with 0-based (row, col) at its centre.""" + half = STAMP // 2 + padded = np.pad(array, half, constant_values=fill) + return padded[row:row + STAMP, col:col + STAMP] + + def _scene(tmp_path, flat=False): - """A sexcat whose VIGNETs are cut as SExtractor cuts them. + """A sexcat whose VIGNETs are cut on the hand-written centres. ``flat`` makes the image uniform, so no pixel comparison can tell a tie object's two candidate centres apart. @@ -44,16 +71,14 @@ def _scene(tmp_path, flat=False): image = (np.full((NY, NX), 100, np.float32) if flat else rng.normal(100, 5, (NY, NX)).astype(np.float32)) seg = np.zeros((NY, NX), np.int32) - for number, x, y in OBJECTS: - col, row = int(np.rint(x - 1)), int(np.rint(y - 1)) + for number, _, _, col, row in OBJECTS: seg[max(row - 2, 0):row + 3, max(col - 2, 0):col + 3] = number number = np.array([o[0] for o in OBJECTS]) x = np.array([o[1] for o in OBJECTS]) y = np.array([o[2] for o in OBJECTS]) - col, row = np.rint(x - 1).astype(int), np.rint(y - 1).astype(int) - vignets = ss.cut_stamps(image, col, row, STAMP, BIG) - seg_true = ss.cut_stamps(seg, col, row, STAMP, 0) + vignets = np.array([_cut(image, o[3], o[4], BIG) for o in OBJECTS]) + seg_true = np.array([_cut(seg, o[3], o[4], 0) for o in OBJECTS]) vignets[(seg_true != 0) & (seg_true != number[:, None, None])] = BIG paths = {name: str(tmp_path / f"{name}-001-001.fits") @@ -66,7 +91,8 @@ def _scene(tmp_path, flat=False): fits.Column(name="VIGNET", format=f"{STAMP * STAMP}E", array=vignets.reshape(len(number), -1), dim=f"({STAMP},{STAMP})"), - fits.Column(name="THETA_J2000", format="E", array=np.zeros(5)), + fits.Column(name="THETA_J2000", format="E", + array=np.zeros(len(number))), fits.Column(name="X_IMAGE_DBL", format="D", array=x), fits.Column(name="Y_IMAGE_DBL", format="D", array=y), ], name="LDAC_OBJECTS") @@ -84,16 +110,20 @@ def _add(paths): return [h.name for h in hdul], hdul["LDAC_OBJECTS"].data.copy() -def test_float32_catalogue_positions_tie_on_half_pixels(): - """The premise: the float32 columns lose which way objects 3 and 4 - rounded, so no rounding rule on the catalogue alone recovers both.""" - x3 = np.float32(OBJECTS[2][1]) - y4 = np.float32(OBJECTS[3][2]) - assert x3 % 1 == 0.5 and y4 % 1 == 0.5 - assert np.rint(OBJECTS[2][1] - 1) == np.floor(x3 - 1) + 1 - assert np.rint(OBJECTS[3][2] - 1) == np.floor(y4 - 1) - assert np.rint(OBJECTS[2][1] - 1) != np.rint(np.float64(x3) - 1) - assert np.rint(OBJECTS[3][2] - 1) != np.rint(np.float64(y4) - 1) +def test_neither_rint_nor_float32_positions_give_the_centres(): + """The premise: np.rint of the double positions misses objects 3 and 4, + and SExtractor's rule on the float32 positions misses object 6.""" + x = np.array([o[1] for o in OBJECTS]) + y = np.array([o[2] for o in OBJECTS]) + col = np.array([o[3] for o in OBJECTS]) + row = np.array([o[4] for o in OBJECTS]) + rint_miss = (np.rint(x - 1) != col) | (np.rint(y - 1) != row) + assert np.flatnonzero(rint_miss).tolist() == [2, 3] + x32 = x.astype(np.float32).astype(np.float64) + y32 = y.astype(np.float32).astype(np.float64) + f32_miss = ((ss.vignet_centre(x32) != col) + | (ss.vignet_centre(y32) != row)) + assert np.flatnonzero(f32_miss).tolist() == [5] @pytest.mark.parametrize("flat", [False, True], ids=["noisy", "flat"]) From 3f5a2633ce04a2df8eaaede12e5147d896c5d110 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 1 Oct 2026 21:38:05 -0400 Subject: [PATCH 44/44] match_catalogue: refuse NUMBERs the int32 columns cannot hold The guard took unique positive NUMBERs but not their type or range: a non-integer, or a value past 2**31 - 1, narrows on the way into the int32 NUMBER and SEG_VIGNET columns (4294967295 becomes -1, UNMATCHED_LABEL). The join now requires integers in [1, MAX_NUMBER]. Boundary tests cover repeats, 0, negatives, 5.5, 5.0, 2**31, 2**32 - 1 and accept 2**31 - 1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017JQtu8oaZQEAqzbxZbPpWs --- .../sextractor_package/match_catalogue.py | 22 +++++--- tests/module/test_sextractor_seg_vignet.py | 50 +++++++++++++------ 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/shapepipe/modules/sextractor_package/match_catalogue.py b/src/shapepipe/modules/sextractor_package/match_catalogue.py index da537026f..6c59d31e7 100644 --- a/src/shapepipe/modules/sextractor_package/match_catalogue.py +++ b/src/shapepipe/modules/sextractor_package/match_catalogue.py @@ -27,6 +27,9 @@ # asks "self or not self". UNMATCHED_LABEL = -1 +# The largest NUMBER the int32 NUMBER and SEG_VIGNET columns hold. +MAX_NUMBER = np.iinfo(np.int32).max + def mutual_nearest(x_a, y_a, x_b, y_b, radius): """One-to-one pairs of mutual nearest neighbours closer than ``radius``. @@ -103,8 +106,9 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98, rows leave the catalogue. A ``SEG_VIGNET`` column, when present, is relabelled through the whole map from old to new numbers, with the footprints of rows that left marked ``UNMATCHED_LABEL``: as the external - numbers are unique and positive, each row's own footprint carries its new - ``NUMBER`` and no other footprint can, whatever the two numberings share. + numbers are unique integers in ``[1, MAX_NUMBER]``, each row's own + footprint carries its new ``NUMBER`` and no other footprint can, + whatever the two numberings share. The catalogue is rewritten in place; every other HDU and column is kept. Both catalogues come from the same pixels, so pairs agree to ~1e-4 @@ -147,7 +151,8 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98, Raises ------ ValueError - If the external ``NUMBER`` repeats or is not positive, or if more + If the external ``NUMBER`` repeats or is not an integer in + ``[1, MAX_NUMBER]``, or if more than ``tolerated_unpaired`` rows of either side, and more than ``1 - min_fraction`` of it, have no partner @@ -156,11 +161,14 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98, ext = asc.read(ext_cat_path, format="sextractor", include_names=["NUMBER", "X_IMAGE", "Y_IMAGE"]) ext_number = np.asarray(ext["NUMBER"]) - if (ext_number <= 0).any() or len(np.unique(ext_number)) < len(ext): + if (not np.issubdtype(ext_number.dtype, np.integer) + or (ext_number <= 0).any() or (ext_number > MAX_NUMBER).any() + or len(np.unique(ext_number)) < len(ext)): raise ValueError( - f"{ext_cat_path} has a repeated or non-positive NUMBER; the join" - + " needs unique positive numbers, so that no relabelled" - + " SEG_VIGNET footprint takes another object's number." + f"{ext_cat_path} has a NUMBER that is not an integer in" + + f" [1, {MAX_NUMBER}] or that repeats; the join needs unique" + + " numbers that fit the int32 NUMBER and SEG_VIGNET columns, so" + + " that no relabelled footprint takes another object's number." ) with fits.open(cat_path) as hdul: hdus = [hdu.copy() for hdu in hdul] diff --git a/tests/module/test_sextractor_seg_vignet.py b/tests/module/test_sextractor_seg_vignet.py index f93e1a04a..f66faa99a 100644 --- a/tests/module/test_sextractor_seg_vignet.py +++ b/tests/module/test_sextractor_seg_vignet.py @@ -398,23 +398,43 @@ def test_a_partial_relabel_would_collide(): assert ((partial == 2) & (old != 1)).any() -def test_join_rejects_a_catalogue_with_repeated_or_nonpositive_numbers( - tmp_path, -): - """Unique positive catalogue NUMBERs are what keep a relabelled stamp's - own footprint apart from every other one.""" +def _join_with_numbers(case, numbers): + """Join CROWD to a catalogue whose three objects carry ``numbers`` + (written as given), and return the joined NUMBER column.""" + from shapepipe.modules.sextractor_package import match_catalogue as mc + + case.mkdir() + _write_crowd(case / "sexcat.fits", case / "seg.fits") + rows = zip(numbers, [10.0, 10.0, 22.0], [10.0, 13.0, 22.0]) + ext = case / "ext.cat" + ext.write_text(EXT_HEADER + "".join( + f"{n:>10} {x:11.4f} {y:11.4f}\n" for n, x, y in rows)) + mc.match_catalogue(str(case / "sexcat.fits"), str(ext)) + with fits.open(case / "sexcat.fits") as hdul: + return hdul["LDAC_OBJECTS"].data["NUMBER"].tolist() + + +@pytest.mark.parametrize("numbers", [ + [5, 5, 6], # repeated + [0, 5, 6], # not positive + [-3, 5, 6], + ["5.5", 7, 6], # not an integer + ["5.0", 7, 6], # read as a float + [2**31, 5, 6], # past int32 + [2**32 - 1, 5, 6], # would narrow to -1, UNMATCHED_LABEL +]) +def test_join_rejects_numbers_that_could_collide(tmp_path, numbers): + """Unique catalogue NUMBERs that the int32 columns hold exactly are what + keep a relabelled stamp's own footprint apart from every other one.""" + with pytest.raises(ValueError, match="NUMBER"): + _join_with_numbers(tmp_path / "case", numbers) + + +def test_join_accepts_the_largest_int32_number(tmp_path): from shapepipe.modules.sextractor_package import match_catalogue as mc - for numbers in ([5, 5, 6], [0, 5, 6]): - case = tmp_path / str(numbers[0] + numbers[1]) - case.mkdir() - _write_crowd(case / "sexcat.fits", case / "seg.fits") - rows = zip(numbers, [10.0, 10.0, 22.0], [10.0, 13.0, 22.0]) - ext = case / "ext.cat" - ext.write_text(EXT_HEADER + "".join( - f"{n:10d} {x:11.4f} {y:11.4f}\n" for n, x, y in rows)) - with pytest.raises(ValueError, match="NUMBER"): - mc.match_catalogue(str(case / "sexcat.fits"), str(ext)) + assert _join_with_numbers(tmp_path / "case", [1, mc.MAX_NUMBER, 6]) == [ + 1, mc.MAX_NUMBER, 6] def test_seg_vignet_stays_out_of_the_final_catalogue(tmp_path):