From 79b20bc941a543b28b8ff2d99aad279054e3f1f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 11:00:17 +0000 Subject: [PATCH 1/5] phase 1 of issue #161 --- mortie/moc.py | 21 ++++---- mortie/tests/test_coverage.py | 33 ++++++++++++ src_rust/src/coverage.rs | 4 +- src_rust/src/coverage/tests.rs | 2 +- src_rust/src/dissolve.rs | 4 +- src_rust/src/dissolve/tests.rs | 2 +- src_rust/src/lib.rs | 13 ++++- src_rust/src/moc.rs | 98 +++++++++++++++++++++++++++------- src_rust/src/moc/batch.rs | 26 ++++----- src_rust/src/wkb/batch.rs | 2 +- 10 files changed, 156 insertions(+), 49 deletions(-) diff --git a/mortie/moc.py b/mortie/moc.py index 470b8f23..1c06a86e 100644 --- a/mortie/moc.py +++ b/mortie/moc.py @@ -68,16 +68,17 @@ def moc_to_order(morton, order, max_cells=_FLAT_COVER_WARN_THRESHOLD): ``order`` (which coarsen and dedup on densify), where it is a safe over-count — so the guard never lets more than ``max_cells`` cells through. - ``order`` is range-checked here, in the wrapper, for the same reason. The - kernel's densify shift is only defined over 0-29; an out-of-range order - reaches it as a Rust panic, surfacing as ``pyo3_runtime.PanicException``, - which derives from :class:`BaseException` — so neither ``except ValueError`` - nor ``except Exception`` catches it. The budget does not screen it either: - the estimate's ``1 << (2 * (order - depth))`` wraps mod 64 in a release - build, so for depth-6 input the whole band ``order`` 38-48 estimates *under* - the default budget and passes through to the panic. Refusing with the - :class:`ValueError` this contract already promises keeps it catchable by the - handlers consumers already have (issue #108). + ``order`` is range-checked here, in the wrapper, for the same reason: the + refusal must be catchable by the handlers consumers already have (issue + #108), and a Rust panic surfaces as ``pyo3_runtime.PanicException``, which + derives from :class:`BaseException` — so neither ``except ValueError`` nor + ``except Exception`` catches it. The kernel's densify shift is defined only + over 0-29, and past that ``1 << (2 * (order - depth))`` used to wrap mod 64 + in a release build rather than trap: for depth-6 input the whole band + ``order`` 38-48 estimated *under* the default budget and passed straight + through to the panic. That shift now refuses out of range in Rust too and + the binding maps it to the same :class:`ValueError` (issue #161), so this + check is defence in depth rather than the only defence. Parameters ---------- diff --git a/mortie/tests/test_coverage.py b/mortie/tests/test_coverage.py index 7d18311b..f3a8d785 100644 --- a/mortie/tests/test_coverage.py +++ b/mortie/tests/test_coverage.py @@ -683,6 +683,39 @@ def test_order_zero_is_in_range(self): dens = mortie.moc_to_order(self.BASE_CELL, 0) np.testing.assert_array_equal(dens, self.BASE_CELL) + def test_kernel_refuses_out_of_range_order_behind_the_wrapper(self): + # The wrapper guard above is a fence; the kernel is now correct on its + # own (issue #161). Call the bindings directly, past the fence, at the + # same orders: `1 << (2 * (order - depth))` used to wrap mod 64 -- a + # depth-6 word estimated 1 cell at order 38 (under budget, straight + # through to the densify's PanicException) and 1125899906842624 at 255. + # Both entry points must raise a ValueError that `except Exception` + # catches, since PanicException derives from BaseException and does not. + from mortie import _rustie + + depth6 = np.atleast_1d(mortie.norm2mort(0, 0, 6)).astype(np.uint64) + for order in (30, 38, 44, 48, 70, 255): + for call in (_rustie.rust_moc_to_order_count, + _rustie.rust_moc_to_order): + try: + call(depth6, order) + except Exception as exc: # must not escape as PanicException + assert isinstance(exc, ValueError), type(exc) + assert f"between 0 and 29, got {order}" in str(exc) + else: + raise AssertionError( + f"{call.__name__} order={order} did not raise") + + def test_kernel_estimate_is_exact_at_the_boundary_order(self): + # The refusal is `order > 29`, inclusive at 29 -- the deepest order the + # packed word represents, not an out-of-range one. A depth-28 word + # densifies to its 4 children, and the estimate agrees exactly. + from mortie import _rustie + + deep = np.atleast_1d(mortie.norm2mort(0, 0, 28)).astype(np.uint64) + assert int(_rustie.rust_moc_to_order_count(deep, 29)) == 4 + assert len(np.asarray(_rustie.rust_moc_to_order(deep, 29))) == 4 + class TestCoverageHighOrder: """Order 19–29 coverage (issue #60). diff --git a/src_rust/src/coverage.rs b/src_rust/src/coverage.rs index b8df2e53..db74d9b5 100644 --- a/src_rust/src/coverage.rs +++ b/src_rust/src/coverage.rs @@ -85,7 +85,7 @@ pub fn polygon_to_morton_coverage( normalize: bool, ) -> Vec { let moc = polygon_descend(lats, lons, order, None, normalize); - crate::moc::to_order(&moc, order) + crate::moc::to_order(&moc, order).expect("polygon_descend asserted order 1-29") } /// Compute polygon coverage as a compact, normalized Multi-Order Coverage map: @@ -157,7 +157,7 @@ pub fn multipolygon_to_morton_coverage( validate_multi(lats, lons, order); let (rings, refs) = build_rings(lats, lons, normalize); let moc = nodes_to_morton(&descend_parallel(&rings, &refs, order, None)); - crate::moc::to_order(&moc, order) + crate::moc::to_order(&moc, order).expect("validate_multi asserted order 1-29") } /// MOC coverage of a ring-set, with optional `tolerance` or `max_cells` stop. diff --git a/src_rust/src/coverage/tests.rs b/src_rust/src/coverage/tests.rs index f33ef779..5ffafc84 100644 --- a/src_rust/src/coverage/tests.rs +++ b/src_rust/src/coverage/tests.rs @@ -255,7 +255,7 @@ fn test_moc_is_compact_and_densifies_to_flat() { "interior should collapse to coarse cells" ); assert_eq!( - crate::moc::to_order(&moc, 8), + crate::moc::to_order(&moc, 8).unwrap(), flat, "MOC must densify to flat" ); diff --git a/src_rust/src/dissolve.rs b/src_rust/src/dissolve.rs index 39664fd0..76ffeb05 100644 --- a/src_rust/src/dissolve.rs +++ b/src_rust/src/dissolve.rs @@ -105,7 +105,7 @@ fn debug_assert_cover_on_left(rings: &[Vec], morton: &[u64]) { let depths: Vec = morton.iter().map(|&w| mort2nested(w).1).collect(); let max_depth = *depths.iter().max().unwrap(); let flat: Vec = if depths.iter().any(|&d| d != max_depth) { - moc::to_order(morton, max_depth) + moc::to_order(morton, max_depth).expect("max_depth decoded from words is <= 29") } else { morton.to_vec() }; @@ -165,7 +165,7 @@ fn survivor_edges(morton: &[u64], step: u32) -> (Vec<(u32, u32)>, Vec) { let max_depth = *depths.iter().max().unwrap(); let min_depth = *depths.iter().min().unwrap(); let flat: Vec = if min_depth != max_depth { - moc::to_order(morton, max_depth) + moc::to_order(morton, max_depth).expect("max_depth decoded from words is <= 29") } else { morton.to_vec() }; diff --git a/src_rust/src/dissolve/tests.rs b/src_rust/src/dissolve/tests.rs index 50455f08..bda82501 100644 --- a/src_rust/src/dissolve/tests.rs +++ b/src_rust/src/dissolve/tests.rs @@ -75,7 +75,7 @@ fn assert_point_sampled(cover: &[u64], out: &ClassifiedRings, sample_order: u8) let depths: Vec = cover.iter().map(|&w| mort2nested(w).1).collect(); let max_depth = *depths.iter().max().unwrap(); let flat: Vec = if depths.iter().any(|&d| d != max_depth) { - moc::to_order(cover, max_depth) + moc::to_order(cover, max_depth).unwrap() } else { cover.to_vec() }; diff --git a/src_rust/src/lib.rs b/src_rust/src/lib.rs index 60e6d808..ea523d4b 100644 --- a/src_rust/src/lib.rs +++ b/src_rust/src/lib.rs @@ -1013,6 +1013,9 @@ fn rust_moc_normalize(py: Python<'_>, morton: PyReadonlyArray1) -> PyResult } /// Densify a (mixed-order) morton set to a flat list at `order`. +/// +/// `order` above 29 raises `ValueError` — the densify shift is undefined there +/// and used to wrap mod 64 into a `PanicException` (issue #161). #[pyfunction] #[pyo3(signature = (morton, order))] fn rust_moc_to_order( @@ -1021,12 +1024,17 @@ fn rust_moc_to_order( order: u8, ) -> PyResult { let data = morton.to_vec()?; - let densified = py.allow_threads(|| moc::to_order(&data, order)); + let densified = py + .allow_threads(|| moc::to_order(&data, order)) + .map_err(PyValueError::new_err)?; Ok(densified.into_pyarray_bound(py).into_any().unbind()) } /// Exact flat cell count `rust_moc_to_order` would produce at `order`, computed /// from the compact MOC without materializing the flat list (issue #80). +/// +/// Shares `rust_moc_to_order`'s `order` domain and raises the same `ValueError` +/// past it, so the guard's estimate can never be a fabricated one (issue #161). #[pyfunction] #[pyo3(signature = (morton, order))] fn rust_moc_to_order_count( @@ -1035,7 +1043,8 @@ fn rust_moc_to_order_count( order: u8, ) -> PyResult { let data = morton.to_vec()?; - Ok(py.allow_threads(|| moc::to_order_count(&data, order))) + py.allow_threads(|| moc::to_order_count(&data, order)) + .map_err(PyValueError::new_err) } /// Densify many (mixed-order) MOCs to a flat `order` in one call (issue #156). diff --git a/src_rust/src/moc.rs b/src_rust/src/moc.rs index 6bd449cd..d70a21e4 100644 --- a/src_rust/src/moc.rs +++ b/src_rust/src/moc.rs @@ -154,12 +154,38 @@ pub fn moc_xor(a: &[u64], b: &[u64]) -> Vec { bmoc_to_morton(build_bmoc(a).xor(&build_bmoc(b))) } +/// Reject a densify target `order` the packed-u64 grid cannot represent +/// (issue #161). +/// +/// The densify arithmetic is a shift by `2 * (order - depth)`, defined only +/// while that stays under 64. `depth` is at most [`MAX_DEPTH`] by decode, so +/// bounding `order` there bounds the shift at 58. Past it a release build +/// **wraps the shift mod 64** rather than trapping: at depth 6, `order` 38 +/// shifts by 0 and `order` 255 by 50, so the count is fabricated (a small one +/// for 38–48, sailing through the caller's budget guard) and the densify then +/// dies in `nested2mort` as a `PanicException` — `BaseException`-derived, so +/// caught by neither `except ValueError` nor `except Exception`. Refusing here +/// keeps the kernel correct for any caller, not just those behind the wrapper +/// guard at `mortie/moc.py`. +fn check_order(order: u8) -> Result<(), String> { + if order > MAX_DEPTH { + return Err(format!( + "Order must be between 0 and {MAX_DEPTH}, got {order}" + )); + } + Ok(()) +} + /// Densify a (possibly mixed-order) morton set to a flat list at `order`. /// /// Cells coarser than `order` are expanded to their `4^(order-depth)` /// descendants; cells already at `order` are kept; cells finer than `order` /// (unusual) are coarsened to their ancestor at `order`. Returns sorted unique. -pub fn to_order(morton: &[u64], order: u8) -> Vec { +/// +/// # Errors +/// `order` above [`MAX_DEPTH`] — see [`check_order`]. +pub fn to_order(morton: &[u64], order: u8) -> Result, String> { + check_order(order)?; let mut out = Vec::with_capacity(morton.len()); for &m in morton { let (nested, depth) = mort2nested(m); @@ -179,7 +205,7 @@ pub fn to_order(morton: &[u64], order: u8) -> Vec { } out.sort_unstable(); out.dedup(); - out + Ok(out) } /// Upper bound on the flat cell count [`to_order`] would produce, from the input @@ -195,7 +221,13 @@ pub fn to_order(morton: &[u64], order: u8) -> Vec { /// to one flat cell while this counts each as 1 — the real count can only be /// smaller, so the guard never lets through more than estimated. Saturates so a /// pathological estimate cannot overflow the guard it feeds. -pub fn to_order_count(morton: &[u64], order: u8) -> u64 { +/// +/// # Errors +/// `order` above [`MAX_DEPTH`] — see [`check_order`]. The estimate is the +/// budget guard's only input, so a fabricated one is worse than no estimate: +/// it under-counts and waves an unrepresentable request through. +pub fn to_order_count(morton: &[u64], order: u8) -> Result { + check_order(order)?; let mut total: u64 = 0; for &m in morton { let (_nested, depth) = mort2nested(m); @@ -206,7 +238,7 @@ pub fn to_order_count(morton: &[u64], order: u8) -> u64 { }; total = total.saturating_add(cells); } - total + Ok(total) } /// Half-open range `[start, end)` a cell covers on the uniform `MAX_DEPTH` grid. @@ -364,7 +396,7 @@ mod tests { fn test_to_order_expands_coarse() { // One cell at depth 2 → 4^(5-2) = 64 leaves at order 5. let coarse = nested2mort(7, 2); - let flat = to_order(&[coarse], 5); + let flat = to_order(&[coarse], 5).unwrap(); assert_eq!(flat.len(), 64); // All leaves must be descendants (same nested prefix at depth 2). for &m in &flat { @@ -377,7 +409,7 @@ mod tests { #[test] fn test_to_order_keeps_same_order() { let cells: Vec = (10..20).map(|n| nested2mort(n, 6)).collect(); - let flat = to_order(&cells, 6); + let flat = to_order(&cells, 6).unwrap(); let mut expected = cells.clone(); expected.sort_unstable(); assert_eq!(flat, expected); @@ -392,8 +424,8 @@ mod tests { nested2mort(10, 5), // at order -> 1 nested2mort(11, 5), // at order -> 1 ]; - let estimate = to_order_count(&cover, 5); - let flat = to_order(&cover, 5); + let estimate = to_order_count(&cover, 5).unwrap(); + let flat = to_order(&cover, 5).unwrap(); assert_eq!(estimate, 66); assert_eq!(estimate, flat.len() as u64); } @@ -405,8 +437,8 @@ mod tests { // siblings under one depth-5 ancestor flatten to 1 cell, but the count // adds 1 each (4). estimate >= actual must hold. let cover: Vec = (0..4).map(|s| nested2mort((9 << 2) | s, 6)).collect(); - let estimate = to_order_count(&cover, 5); - let flat = to_order(&cover, 5); + let estimate = to_order_count(&cover, 5).unwrap(); + let flat = to_order(&cover, 5).unwrap(); assert_eq!(estimate, 4, "one per finer cell"); assert_eq!(flat.len(), 1, "all four collapse to ancestor 9@5"); assert!(estimate >= flat.len() as u64, "estimate is an upper bound"); @@ -414,7 +446,32 @@ mod tests { #[test] fn test_to_order_count_empty() { - assert_eq!(to_order_count(&[], 5), 0); + assert_eq!(to_order_count(&[], 5).unwrap(), 0); + } + + #[test] + fn test_out_of_range_order_is_an_error_not_a_wrapped_shift() { + // The shift `2 * (order - depth)` wrapped mod 64 in a release build + // (issue #161). For this depth-6 cell: order 38 wrapped to a shift of + // 0 (count 1, so a caller's budget waved it through to a panic in + // `nested2mort`), order 255 to a shift of 50 (a fabricated + // 1125899906842624). Both must be `Err` now, at every order the + // Python-side test covers. + let cell = [nested2mort(0, 6)]; + for order in [30u8, 38, 44, 48, 70, 255] { + let err = to_order_count(&cell, order).unwrap_err(); + assert!(err.contains("between 0 and 29"), "order={order}: {err}"); + assert!(to_order(&cell, order).is_err(), "order={order} densified"); + } + } + + #[test] + fn test_max_depth_order_still_densifies() { + // The guard's boundary is inclusive: order 29 is the deepest shift the + // packed word represents (2 * (29 - 28) here), not an out-of-range one. + let cell = [nested2mort(0, 28)]; + assert_eq!(to_order_count(&cell, 29).unwrap(), 4); + assert_eq!(to_order(&cell, 29).unwrap().len(), 4); } #[test] @@ -448,8 +505,8 @@ mod tests { fn test_normalize_then_to_order_roundtrip() { // Densify-invariance: normalizing must not change the flattened cover. let children: Vec = (0..4).map(|s| nested2mort((9 << 2) | s, 5)).collect(); - let direct = to_order(&children, 5); - let viamoc = to_order(&normalize(&children), 5); + let direct = to_order(&children, 5).unwrap(); + let viamoc = to_order(&normalize(&children), 5).unwrap(); assert_eq!(direct, viamoc); } @@ -593,8 +650,8 @@ mod tests { /// must be >= the deepest cell in either input for the result to be exact. fn setop_reference(a: &[u64], b: &[u64], order: u8, op: fn(bool, bool) -> bool) -> Vec { use std::collections::BTreeSet; - let la: BTreeSet = to_order(a, order).into_iter().collect(); - let lb: BTreeSet = to_order(b, order).into_iter().collect(); + let la: BTreeSet = to_order(a, order).unwrap().into_iter().collect(); + let lb: BTreeSet = to_order(b, order).unwrap().into_iter().collect(); let mut out: Vec = la .union(&lb) .filter(|&&m| op(la.contains(&m), lb.contains(&m))) @@ -667,7 +724,8 @@ mod tests { // The two inside cells (5,6 @4) cancel against base0's coverage; 300 // (outside base cell 0) survives. Densify to depth 4 and check exactly: // 300 present, 5 and 6 absent. - let leaves: std::collections::BTreeSet = to_order(&got, 4).into_iter().collect(); + let leaves: std::collections::BTreeSet = + to_order(&got, 4).unwrap().into_iter().collect(); assert!( leaves.contains(&nested2mort(300, 4)), "outside cell must survive" @@ -941,11 +999,15 @@ mod tests { // and is the 5 shared cells (5..10); densifying back to `order` must // recover exactly those, proving the deep BMOC round-trip is lossless. let shared: Vec = (5..10).map(|n| nested2mort(origin + n, order)).collect(); - assert_eq!(to_order(&moc_and(&a, &b), order), shared, "and @ {order}"); + assert_eq!( + to_order(&moc_and(&a, &b), order).unwrap(), + shared, + "and @ {order}" + ); // a \ b is the 5 cells only in a (0..5). let only_a: Vec = (0..5).map(|n| nested2mort(origin + n, order)).collect(); assert_eq!( - to_order(&moc_minus(&a, &b), order), + to_order(&moc_minus(&a, &b), order).unwrap(), only_a, "minus @ {order}" ); diff --git a/src_rust/src/moc/batch.rs b/src_rust/src/moc/batch.rs index 1db03ad8..81fe57ad 100644 --- a/src_rust/src/moc/batch.rs +++ b/src_rust/src/moc/batch.rs @@ -122,7 +122,7 @@ fn validate_batch( )); } if let Some(budget) = max_cells { - let estimated = to_order_count(&values[s as usize..e as usize], order); + let estimated = to_order_count(&values[s as usize..e as usize], order)?; if estimated > budget { return Err(format!( "moc {i}: moc_to_order would densify to ~{estimated} cells at \ @@ -203,12 +203,14 @@ impl BatchOrders { /// Run one MOC's kernel, turning a panic into a MOC-named error. /// -/// For the densify this is defensive — `validate_batch` screens the one input -/// [`to_order`] cannot take (an out-of-range `order`), so no known input -/// reaches its panic arm. For the set ops it is a **live** path: layout -/// validation does not screen the morton words themselves, so a malformed -/// word in an item (e.g. the empty word 0) panics in `mort2nested` and -/// surfaces here as a `ValueError` naming that item. Both regimes are +/// For the densify this is defensive — [`to_order`] no longer panics on the one +/// input it cannot take (an out-of-range `order`): it returns `Err` (issue +/// #161), which is threaded through under the same MOC-named prefix, and +/// `validate_batch` refuses that order ahead of the parallel pass anyway. For +/// the set ops it is a **live** path: layout validation does not screen the +/// morton words themselves, so a malformed word in an item (e.g. the empty +/// word 0) panics in `mort2nested` and surfaces here as a `ValueError` naming +/// that item. Both regimes are /// tested: injected panicking kernels pin the capture-and-name mechanism, and /// a malformed-word test drives the real rayon path across a chunk seam. fn run_moc(i: usize, kernel: F) -> Result @@ -256,7 +258,7 @@ pub fn mocs_to_orders( .into_par_iter() .map(|i| { let (s, e) = (offsets[i] as usize, offsets[i + 1] as usize); - run_moc(i, || to_order(&values[s..e], order)) + run_moc(i, || to_order(&values[s..e], order))?.map_err(|e| format!("moc {i}: {e}")) }) .collect(); out.extend_chunk(flats)?; @@ -416,7 +418,7 @@ mod tests { assert_eq!(*out.offsets.last().unwrap() as usize, out.values.len()); for i in 0..3 { let (s, e) = (offsets[i] as usize, offsets[i + 1] as usize); - let scalar = to_order(&values[s..e], 8); + let scalar = to_order(&values[s..e], 8).unwrap(); let got = &out.values[out.offsets[i] as usize..out.offsets[i + 1] as usize]; assert_eq!(got, &scalar[..]); } @@ -434,7 +436,7 @@ mod tests { let out = mocs_to_orders(&values, &offsets, 7, None).unwrap(); assert_eq!(out.offsets.len(), n + 1); for i in 0..n { - let scalar = to_order(&values[i..i + 1], 7); + let scalar = to_order(&values[i..i + 1], 7).unwrap(); let got = &out.values[out.offsets[i] as usize..out.offsets[i + 1] as usize]; assert_eq!(got, &scalar[..]); } @@ -492,7 +494,7 @@ mod tests { assert!(err.contains("must end at the value count"), "{err}"); // The exactly-covering spelling of that first MOC is accepted. let out = mocs_to_orders(&values[..1], &[0, 1], 8, None).unwrap(); - assert_eq!(out.values, to_order(&values[..1], 8)); + assert_eq!(out.values, to_order(&values[..1], 8).unwrap()); } #[test] @@ -540,7 +542,7 @@ mod tests { let out = mocs_to_orders(&values, &offsets, 0, None).unwrap(); for i in 0..3 { let (s, e) = (offsets[i] as usize, offsets[i + 1] as usize); - let scalar = to_order(&values[s..e], 0); + let scalar = to_order(&values[s..e], 0).unwrap(); let got = &out.values[out.offsets[i] as usize..out.offsets[i + 1] as usize]; assert_eq!(got, &scalar[..]); } diff --git a/src_rust/src/wkb/batch.rs b/src_rust/src/wkb/batch.rs index bf79d868..35962490 100644 --- a/src_rust/src/wkb/batch.rs +++ b/src_rust/src/wkb/batch.rs @@ -226,7 +226,7 @@ mod tests { // smaller region (its *MOC* is longer, since a hole fragments the // compact spelling, so compare covered cells rather than words). let solid = run(&[polygon(&[quad(0.0, 0.0)])], 8); - let covered = |m: &[u64]| crate::moc::to_order(m, 8).len(); + let covered = |m: &[u64]| crate::moc::to_order(m, 8).unwrap().len(); assert!(covered(&got[0].as_ref().unwrap().0) < covered(&solid[0].as_ref().unwrap().0)); } From 52b1f59f13e9da3996869707665109b314803fa7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 11:12:36 +0000 Subject: [PATCH 2/5] fold review: per-MOC error prefix, run_moc and estimate doc accuracy (issue #161) --- src_rust/src/lib.rs | 4 +++- src_rust/src/moc/batch.rs | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src_rust/src/lib.rs b/src_rust/src/lib.rs index ea523d4b..cb36f5df 100644 --- a/src_rust/src/lib.rs +++ b/src_rust/src/lib.rs @@ -1034,7 +1034,9 @@ fn rust_moc_to_order( /// from the compact MOC without materializing the flat list (issue #80). /// /// Shares `rust_moc_to_order`'s `order` domain and raises the same `ValueError` -/// past it, so the guard's estimate can never be a fabricated one (issue #161). +/// past it, so no `order` can make the guard's estimate a fabricated one +/// (issue #161). A malformed *word* is a separate matter — it still panics in +/// `mort2nested`, as it does on every kernel that decodes one. #[pyfunction] #[pyo3(signature = (morton, order))] fn rust_moc_to_order_count( diff --git a/src_rust/src/moc/batch.rs b/src_rust/src/moc/batch.rs index 81fe57ad..255922bc 100644 --- a/src_rust/src/moc/batch.rs +++ b/src_rust/src/moc/batch.rs @@ -122,7 +122,8 @@ fn validate_batch( )); } if let Some(budget) = max_cells { - let estimated = to_order_count(&values[s as usize..e as usize], order)?; + let estimated = to_order_count(&values[s as usize..e as usize], order) + .map_err(|e| format!("moc {i}: {e}"))?; if estimated > budget { return Err(format!( "moc {i}: moc_to_order would densify to ~{estimated} cells at \ @@ -203,16 +204,17 @@ impl BatchOrders { /// Run one MOC's kernel, turning a panic into a MOC-named error. /// -/// For the densify this is defensive — [`to_order`] no longer panics on the one -/// input it cannot take (an out-of-range `order`): it returns `Err` (issue -/// #161), which is threaded through under the same MOC-named prefix, and -/// `validate_batch` refuses that order ahead of the parallel pass anyway. For -/// the set ops it is a **live** path: layout validation does not screen the -/// morton words themselves, so a malformed word in an item (e.g. the empty -/// word 0) panics in `mort2nested` and surfaces here as a `ValueError` naming -/// that item. Both regimes are -/// tested: injected panicking kernels pin the capture-and-name mechanism, and -/// a malformed-word test drives the real rayon path across a chunk seam. +/// The capture is **live** for every kernel here, densify included: layout +/// validation does not screen the morton words themselves, so a malformed word +/// in an item (e.g. the empty word 0) panics in `mort2nested` — inside +/// [`to_order`] as much as inside the set ops — and surfaces here as a +/// `ValueError` naming that item. What it no longer has to catch is an +/// out-of-range `order`: [`to_order`] returns `Err` for that now (issue #161), +/// threaded through under the same MOC-named prefix, and `validate_batch` +/// refuses it ahead of the parallel pass regardless. Tested from both sides: +/// injected panicking kernels pin the capture-and-name mechanism, and +/// `malformed_word_names_lowest_index_across_chunks` drives the real rayon path +/// across a chunk seam. fn run_moc(i: usize, kernel: F) -> Result where F: FnOnce() -> T, From 9a352711d8127a90c8111e277dea32e0f513c7b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 11:18:02 +0000 Subject: [PATCH 3/5] phase 2 of issue #162 --- mortie/batch.py | 19 ++++++++++++++++--- src_rust/src/coverage/batch.rs | 18 ++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/mortie/batch.py b/mortie/batch.py index ce8f68c4..eada265c 100644 --- a/mortie/batch.py +++ b/mortie/batch.py @@ -53,9 +53,22 @@ def polygons_to_morton_mocs(lats, lons, offsets, order=18, tolerance=None, input polygon — against the many→one union of the multipart form. Polygons are covered in chunks and each chunk is copied into the ragged - output as it lands, so peak memory is about the returned ``values`` array - plus one chunk of in-flight covers — not the ~2.5x of holding every - polygon's cover to concatenate at the end. + output as it lands, so the per-polygon covers never all coexist — not the + ~2.5x of holding every one of them to concatenate at the end. Peak is then + **the input copy + the result + one chunk**: the binding copies ``lats``, + ``lons`` and ``offsets`` before releasing the GIL (a borrowed numpy slice + cannot cross ``allow_threads``), so the vertex arrays are a full second + resident copy for the duration, and nothing short of giving up the GIL + release removes them. Measured on a cold call over synthetic ~1 degree + footprints at order 8, sampling ``/proc/self/statm`` while the call runs: + 100k footprints is 6.9 MiB of input and a 12.9 MiB result behind a + 21.9 MiB peak, and 555,867 (the ATL03 catalog scale) is 38.2 MiB of input + and a 71.6 MiB result behind a 112.0 MiB peak — **1.70x and 1.56x the + result alone**, but 1.11x and 1.02x of ``input + result``. The copy is + gentler here than on :func:`mocs_to_orders`, where coarsening can leave the + result 60x under the peak (vertices are f64 and no coarsen direction can + shrink a cover to nothing), but it is not a rounding error. Size a worker + off ``input + result``, not the result alone. Input and output are ragged arrays in arrow list layout: polygon ``i`` is ``lats[offsets[i]:offsets[i+1]]`` / ``lons[offsets[i]:offsets[i+1]]``, and diff --git a/src_rust/src/coverage/batch.rs b/src_rust/src/coverage/batch.rs index df0d21a0..82a379f9 100644 --- a/src_rust/src/coverage/batch.rs +++ b/src_rust/src/coverage/batch.rs @@ -15,8 +15,22 @@ //! //! Polygons are covered a chunk at a time and each chunk is copied into the //! ragged output as it lands, so the whole batch's per-polygon covers never -//! coexist with the concatenated result — peak ≈ result + one chunk, against -//! the ~2.5x of a cover-everything-then-concatenate pass. +//! coexist with the concatenated result — against the ~2.5x of a +//! cover-everything-then-concatenate pass. +//! +//! Peak is therefore **input copy + result + one chunk** (issue #162). The +//! input term is the pyfunction's `to_vec()`: a `&[f64]` borrowed from numpy +//! cannot cross `py.allow_threads` (it fails the `Ungil` bound), so `lats`, +//! `lons` and `offsets` are each copied whole before the GIL is released and +//! stay resident for the call. Nothing short of giving up the GIL release +//! removes it — it is a floor, not an inefficiency. Measured on a cold call +//! over synthetic ~1° footprints at order 8: 100k is 6.9 MiB of input and a +//! 12.9 MiB result behind a 21.9 MiB peak, and 555,867 (catalog scale) is +//! 38.2 MiB of input and a 71.6 MiB result behind a 112.0 MiB peak — 1.70x and +//! 1.56x the *result alone*, but 1.11x and 1.02x of `input + result`. The +//! copy is gentler here than on [`crate::moc::batch`]'s densify, where +//! coarsening reaches 60x (vertices are f64 and no coarsen direction can shrink +//! a cover to nothing), but it is not a rounding error. //! //! # Error posture //! From 0322b2f1158b822b8339cb9d00f9d4359f6e724b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 11:33:02 +0000 Subject: [PATCH 4/5] fold review: stale CHUNK peak claim, conversion-copy floor, run_moc serial gap (issue #162) --- mortie/batch.py | 24 ++++++++++++++---------- src_rust/src/coverage/batch.rs | 13 ++++++++----- src_rust/src/moc/batch.rs | 28 ++++++++++++++++++---------- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/mortie/batch.py b/mortie/batch.py index eada265c..c7013e49 100644 --- a/mortie/batch.py +++ b/mortie/batch.py @@ -59,16 +59,20 @@ def polygons_to_morton_mocs(lats, lons, offsets, order=18, tolerance=None, ``lons`` and ``offsets`` before releasing the GIL (a borrowed numpy slice cannot cross ``allow_threads``), so the vertex arrays are a full second resident copy for the duration, and nothing short of giving up the GIL - release removes them. Measured on a cold call over synthetic ~1 degree - footprints at order 8, sampling ``/proc/self/statm`` while the call runs: - 100k footprints is 6.9 MiB of input and a 12.9 MiB result behind a - 21.9 MiB peak, and 555,867 (the ATL03 catalog scale) is 38.2 MiB of input - and a 71.6 MiB result behind a 112.0 MiB peak — **1.70x and 1.56x the - result alone**, but 1.11x and 1.02x of ``input + result``. The copy is - gentler here than on :func:`mocs_to_orders`, where coarsening can leave the - result 60x under the peak (vertices are f64 and no coarsen direction can - shrink a cover to nothing), but it is not a rounding error. Size a worker - off ``input + result``, not the result alone. + release removes it. That is a **floor**, not the whole cost: passing + anything this wrapper has to convert — a list, ``float32``, a + non-contiguous slice — adds the conversion's own copy on top, so hand it + contiguous ``float64``/``int64`` arrays to pay the copy only once. + Measured on a cold call over synthetic ~1 degree footprints at order 8, + sampling ``/proc/self/statm`` while the call runs: 100k footprints is + 6.9 MiB of input and a 12.9 MiB result behind a 21.9 MiB peak, and 555,867 + (the ATL03 catalog scale) is 38.2 MiB of input and a 71.6 MiB result behind + a 112.0 MiB peak — **1.70x and 1.56x the result alone**, but 1.11x and + 1.02x of ``input + result``. Those ratios stay near 1 because covering has + no coarsen direction that can shrink the result to nothing; + :func:`mocs_to_orders` does, and reaches 60x there. Near 1 is not + negligible, though — size a worker off ``input + result``, not the result + alone. Input and output are ragged arrays in arrow list layout: polygon ``i`` is ``lats[offsets[i]:offsets[i+1]]`` / ``lons[offsets[i]:offsets[i+1]]``, and diff --git a/src_rust/src/coverage/batch.rs b/src_rust/src/coverage/batch.rs index 82a379f9..fc6b69f1 100644 --- a/src_rust/src/coverage/batch.rs +++ b/src_rust/src/coverage/batch.rs @@ -27,10 +27,11 @@ //! over synthetic ~1° footprints at order 8: 100k is 6.9 MiB of input and a //! 12.9 MiB result behind a 21.9 MiB peak, and 555,867 (catalog scale) is //! 38.2 MiB of input and a 71.6 MiB result behind a 112.0 MiB peak — 1.70x and -//! 1.56x the *result alone*, but 1.11x and 1.02x of `input + result`. The -//! copy is gentler here than on [`crate::moc::batch`]'s densify, where -//! coarsening reaches 60x (vertices are f64 and no coarsen direction can shrink -//! a cover to nothing), but it is not a rounding error. +//! 1.56x the *result alone*, but 1.11x and 1.02x of `input + result`. Those +//! ratios stay near 1 because this path has no coarsen direction that can +//! shrink the result to nothing — [`crate::moc::batch`]'s densify does, and +//! reaches 60x there — but near 1 is not the same as negligible, and the copy +//! is the difference between the two models above. //! //! # Error posture //! @@ -56,7 +57,9 @@ use super::{polygon_to_morton_moc, polygon_to_morton_moc_budget, polygon_to_mort /// output buffer is filled (review of issue #153). Large enough that the /// per-chunk fork-join is noise against the covering work: a sweep of /// 1024/2048/8192 on 100k order-8 footprints put 2048 at the pre-chunking -/// throughput with the peak still ~1.1x the result. +/// throughput with the peak still ~1.1x of `input + result` (the module +/// header's model; that sweep quoted it against the result alone, which omits +/// the input copy — issue #162). pub(crate) const CHUNK: usize = 2048; /// A batch MOC build: ragged `values` / `offsets` (arrow list layout) plus the diff --git a/src_rust/src/moc/batch.rs b/src_rust/src/moc/batch.rs index 255922bc..b94abe5d 100644 --- a/src_rust/src/moc/batch.rs +++ b/src_rust/src/moc/batch.rs @@ -123,7 +123,7 @@ fn validate_batch( } if let Some(budget) = max_cells { let estimated = to_order_count(&values[s as usize..e as usize], order) - .map_err(|e| format!("moc {i}: {e}"))?; + .map_err(|msg| format!("moc {i}: {msg}"))?; if estimated > budget { return Err(format!( "moc {i}: moc_to_order would densify to ~{estimated} cells at \ @@ -204,17 +204,24 @@ impl BatchOrders { /// Run one MOC's kernel, turning a panic into a MOC-named error. /// -/// The capture is **live** for every kernel here, densify included: layout -/// validation does not screen the morton words themselves, so a malformed word -/// in an item (e.g. the empty word 0) panics in `mort2nested` — inside -/// [`to_order`] as much as inside the set ops — and surfaces here as a +/// The capture is **live** for every kernel in the parallel pass, densify +/// included: layout validation does not screen the morton words themselves, so +/// a malformed word in an item (e.g. the empty word 0) panics in `mort2nested` +/// — inside [`to_order`] as much as inside the set ops — and surfaces here as a /// `ValueError` naming that item. What it no longer has to catch is an /// out-of-range `order`: [`to_order`] returns `Err` for that now (issue #161), /// threaded through under the same MOC-named prefix, and `validate_batch` -/// refuses it ahead of the parallel pass regardless. Tested from both sides: -/// injected panicking kernels pin the capture-and-name mechanism, and -/// `malformed_word_names_lowest_index_across_chunks` drives the real rayon path -/// across a chunk seam. +/// refuses it ahead of the parallel pass regardless. +/// +/// It does **not** cover the serial pre-validation: with a `max_cells` budget +/// set, `validate_batch`'s estimate decodes the same words first, outside this +/// `catch_unwind`, so a malformed word there panics out of the call rather than +/// arriving as a named `ValueError` (a pre-existing gap, unrelated to the +/// `order` fix; see the PR thread for issue #161). +/// +/// Tested from both sides: injected panicking kernels pin the capture-and-name +/// mechanism, and `malformed_word_names_lowest_index_across_chunks` drives the +/// real rayon path across a chunk seam for the set ops. fn run_moc(i: usize, kernel: F) -> Result where F: FnOnce() -> T, @@ -260,7 +267,8 @@ pub fn mocs_to_orders( .into_par_iter() .map(|i| { let (s, e) = (offsets[i] as usize, offsets[i + 1] as usize); - run_moc(i, || to_order(&values[s..e], order))?.map_err(|e| format!("moc {i}: {e}")) + run_moc(i, || to_order(&values[s..e], order))? + .map_err(|msg| format!("moc {i}: {msg}")) }) .collect(); out.extend_chunk(flats)?; From 8060dad0700f569e3621b6ced33ffcf2d705b5e3 Mon Sep 17 00:00:00 2001 From: espg Date: Sun, 16 Aug 2026 16:05:52 -0700 Subject: [PATCH 5/5] fold review: run the densify budget estimate under the panic capture (issue #161) --- mortie/tests/test_coverage.py | 21 ++++++++++++++++ mortie/tests/test_moc_batch.py | 33 ++++++++++++++++++++++++ src_rust/src/lib.rs | 16 ++++++++---- src_rust/src/moc/batch.rs | 46 ++++++++++++++++++++++++++-------- 4 files changed, 101 insertions(+), 15 deletions(-) diff --git a/mortie/tests/test_coverage.py b/mortie/tests/test_coverage.py index f3a8d785..2ac4ee5e 100644 --- a/mortie/tests/test_coverage.py +++ b/mortie/tests/test_coverage.py @@ -706,6 +706,27 @@ def test_kernel_refuses_out_of_range_order_behind_the_wrapper(self): raise AssertionError( f"{call.__name__} order={order} did not raise") + def test_malformed_word_is_a_catchable_valueerror(self): + # The word half of the same contract (issue #161 review): the empty + # word 0 panics in `mort2nested`, and neither binding converted it -- + # so `moc_to_order` on it raised PanicException on the default budget + # (the estimate decodes first) and again with max_cells=None (the + # densify decodes). Both are now captured into a plain ValueError. + from mortie import _rustie + + bad = np.zeros(1, dtype=np.uint64) + for max_cells in (1 << 20, None): + try: + mortie.moc_to_order(bad, 5, max_cells=max_cells) + except Exception as exc: # must not escape as PanicException + assert isinstance(exc, ValueError), type(exc) + assert "Morton index cannot be zero" in str(exc) + else: + raise AssertionError(f"max_cells={max_cells} did not raise") + for call in (_rustie.rust_moc_to_order_count, _rustie.rust_moc_to_order): + with pytest.raises(ValueError, match="Morton index cannot be zero"): + call(bad, 5) + def test_kernel_estimate_is_exact_at_the_boundary_order(self): # The refusal is `order > 29`, inclusive at 29 -- the deepest order the # packed word represents, not an out-of-range one. A depth-28 word diff --git a/mortie/tests/test_moc_batch.py b/mortie/tests/test_moc_batch.py index 57514949..0b200495 100644 --- a/mortie/tests/test_moc_batch.py +++ b/mortie/tests/test_moc_batch.py @@ -394,6 +394,39 @@ def test_budget_refusal_precedes_any_densify(): mortie.mocs_to_orders(values, [0, 1, 2], 29) +def test_malformed_word_is_a_named_value_error_at_both_budgets(): + """A bad word is the same named ``ValueError`` with the budget on or off. + + The budget estimate decodes every word in the serial pre-pass, so it panics + in ``mort2nested`` on the empty word 0 exactly as the densify kernel does. + That pass ran outside the kernel's panic capture, so the *default* + ``max_cells`` turned the refusal into a ``pyo3_runtime.PanicException`` -- + ``BaseException``-derived, missed by ``except ValueError`` *and* by + ``except Exception`` -- while ``max_cells=None`` gave the documented + ``ValueError``. Same input, opposite exception contract, chosen by a + keyword default (issue #161). + """ + from mortie import coverage + + values = np.concatenate([np.zeros(1, np.uint64), _word(600, 6)]) + for max_cells in (coverage._FLAT_COVER_WARN_THRESHOLD, None): + with pytest.raises(ValueError, match=r"moc 0: Morton index cannot be zero"): + mortie.mocs_to_orders(values, [0, 2], 8, max_cells=max_cells) + # The default is that same budget, and must behave identically. + with pytest.raises(ValueError, match=r"moc 0: Morton index cannot be zero"): + mortie.mocs_to_orders(values, [0, 2], 8) + # A plain ``except Exception`` must see it -- the PanicException lesson. + caught = None + try: + mortie.mocs_to_orders(values, [0, 2], 8) + except Exception as exc: + caught = exc + assert isinstance(caught, ValueError), caught + # The lowest-index rule holds when the bad word is not MOC 0. + with pytest.raises(ValueError, match=r"moc 1: Morton index cannot be zero"): + mortie.mocs_to_orders(values[::-1].copy(), [0, 1, 2], 8) + + # --------------------------------------------------------------------------- # GIL release # --------------------------------------------------------------------------- diff --git a/src_rust/src/lib.rs b/src_rust/src/lib.rs index cb36f5df..9b5ae2d5 100644 --- a/src_rust/src/lib.rs +++ b/src_rust/src/lib.rs @@ -1015,7 +1015,9 @@ fn rust_moc_normalize(py: Python<'_>, morton: PyReadonlyArray1) -> PyResult /// Densify a (mixed-order) morton set to a flat list at `order`. /// /// `order` above 29 raises `ValueError` — the densify shift is undefined there -/// and used to wrap mod 64 into a `PanicException` (issue #161). +/// and used to wrap mod 64 into a `PanicException` (issue #161). A malformed +/// *word* is a `ValueError` too: the decode panic in `mort2nested` is captured +/// here, the way `rust_mort2nested` captures its own. #[pyfunction] #[pyo3(signature = (morton, order))] fn rust_moc_to_order( @@ -1025,7 +1027,8 @@ fn rust_moc_to_order( ) -> PyResult { let data = morton.to_vec()?; let densified = py - .allow_threads(|| moc::to_order(&data, order)) + .allow_threads(|| std::panic::catch_unwind(|| moc::to_order(&data, order))) + .map_err(|e| PyValueError::new_err(panic_msg(e, "moc_to_order panicked")))? .map_err(PyValueError::new_err)?; Ok(densified.into_pyarray_bound(py).into_any().unbind()) } @@ -1035,8 +1038,10 @@ fn rust_moc_to_order( /// /// Shares `rust_moc_to_order`'s `order` domain and raises the same `ValueError` /// past it, so no `order` can make the guard's estimate a fabricated one -/// (issue #161). A malformed *word* is a separate matter — it still panics in -/// `mort2nested`, as it does on every kernel that decodes one. +/// (issue #161). A malformed *word* raises `ValueError` here as well: the +/// estimate decodes every word, so it runs under the same panic capture the +/// densify does — the guard cannot turn a bad word into a `PanicException` +/// that `except ValueError` misses. #[pyfunction] #[pyo3(signature = (morton, order))] fn rust_moc_to_order_count( @@ -1045,7 +1050,8 @@ fn rust_moc_to_order_count( order: u8, ) -> PyResult { let data = morton.to_vec()?; - py.allow_threads(|| moc::to_order_count(&data, order)) + py.allow_threads(|| std::panic::catch_unwind(|| moc::to_order_count(&data, order))) + .map_err(|e| PyValueError::new_err(panic_msg(e, "moc_to_order_count panicked")))? .map_err(PyValueError::new_err) } diff --git a/src_rust/src/moc/batch.rs b/src_rust/src/moc/batch.rs index b94abe5d..61130f14 100644 --- a/src_rust/src/moc/batch.rs +++ b/src_rust/src/moc/batch.rs @@ -89,9 +89,11 @@ pub struct BatchOrders { /// The budget lives here, not in the parallel pass, for the same reason the /// scalar guard is pre-emptive (issue #80): the estimate is an O(n) pass over /// the input words with no flat allocation, so a refusal happens before any -/// memory is committed. Per-MOC errors name the offending MOC, and are raised -/// ahead of the endpoint check so the lowest-index MOC is still what a caller -/// sees first. +/// memory is committed. The estimate decodes every word, so it runs under +/// [`run_moc`] like any kernel — a malformed word is the same named +/// `ValueError` here as in the parallel pass, not a `PanicException` (issue +/// #161). Per-MOC errors name the offending MOC, and are raised ahead of the +/// endpoint check so the lowest-index MOC is still what a caller sees first. fn validate_batch( values: &[u64], offsets: &[i64], @@ -122,7 +124,7 @@ fn validate_batch( )); } if let Some(budget) = max_cells { - let estimated = to_order_count(&values[s as usize..e as usize], order) + let estimated = run_moc(i, || to_order_count(&values[s as usize..e as usize], order))? .map_err(|msg| format!("moc {i}: {msg}"))?; if estimated > budget { return Err(format!( @@ -213,15 +215,16 @@ impl BatchOrders { /// threaded through under the same MOC-named prefix, and `validate_batch` /// refuses it ahead of the parallel pass regardless. /// -/// It does **not** cover the serial pre-validation: with a `max_cells` budget -/// set, `validate_batch`'s estimate decodes the same words first, outside this -/// `catch_unwind`, so a malformed word there panics out of the call rather than -/// arriving as a named `ValueError` (a pre-existing gap, unrelated to the -/// `order` fix; see the PR thread for issue #161). +/// It covers the **serial** pre-validation too: with a `max_cells` budget set, +/// `validate_batch`'s estimate decodes the same words first, and that call is +/// routed through here as well (issue #161), so a malformed word arrives as the +/// same named `ValueError` whether or not a budget is in play — the default +/// `max_cells` in `mortie/batch.py` used to make it a `PanicException`. /// /// Tested from both sides: injected panicking kernels pin the capture-and-name /// mechanism, and `malformed_word_names_lowest_index_across_chunks` drives the -/// real rayon path across a chunk seam for the set ops. +/// real rayon path across a chunk seam for the set ops, with +/// `malformed_word_in_the_budget_estimate_is_named` covering the serial one. fn run_moc(i: usize, kernel: F) -> Result where F: FnOnce() -> T, @@ -495,6 +498,29 @@ mod tests { assert!(err.starts_with("moc 0:"), "{err}"); } + #[test] + fn malformed_word_in_the_budget_estimate_is_named() { + // The estimate decodes every word, so the empty word 0 panics in + // `mort2nested` in the serial pre-pass — before the parallel one ever + // runs. With the estimate outside `run_moc` that panic escaped the + // call (a `PanicException` in Python) on the *default* budget, while + // `max_cells=None` reached the kernel and named the MOC. Both spellings + // must give the same named error. + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); // keep the test output clean + let values = vec![0u64, nested2mort(1, 6)]; + let with_budget = mocs_to_orders(&values, &[0, 1, 2], 8, Some(1 << 20)); + let without = mocs_to_orders(&values, &[0, 1, 2], 8, None); + std::panic::set_hook(hook); + + let err = with_budget.unwrap_err(); + assert!( + err.starts_with("moc 0: Morton index cannot be zero"), + "{err}" + ); + assert_eq!(err, without.unwrap_err()); + } + #[test] fn offsets_must_exactly_cover_the_values() { let (values, offsets) = ragged();