small fixes: moc.rs densify shift wrap (#161) and the batch memory posture's missing input copy (#162)

mortie/batch.py

@@ -53,9 +53,26 @@ def polygons_to_morton_mocs(lats, lons, offsets, order=18, tolerance=None,
     input polygon — against the many→one union of the multipart form.
 
     Polygons are covered in chunks and each chunk is copied into the ragged
-    output as it lands, so peak memory is about the returned ``values`` array
-    plus one chunk of in-flight covers — not the ~2.5x of holding every
-    polygon's cover to concatenate at the end.
+    output as it lands, so the per-polygon covers never all coexist — not the
+    ~2.5x of holding every one of them to concatenate at the end.  Peak is then
+    **the input copy + the result + one chunk**: the binding copies ``lats``,
+    ``lons`` and ``offsets`` before releasing the GIL (a borrowed numpy slice
+    cannot cross ``allow_threads``), so the vertex arrays are a full second
+    resident copy for the duration, and nothing short of giving up the GIL
+    release removes it.  That is a **floor**, not the whole cost: passing
+    anything this wrapper has to convert — a list, ``float32``, a
+    non-contiguous slice — adds the conversion's own copy on top, so hand it
+    contiguous ``float64``/``int64`` arrays to pay the copy only once.
+    Measured on a cold call over synthetic ~1 degree footprints at order 8,
+    sampling ``/proc/self/statm`` while the call runs: 100k footprints is
+    6.9 MiB of input and a 12.9 MiB result behind a 21.9 MiB peak, and 555,867
+    (the ATL03 catalog scale) is 38.2 MiB of input and a 71.6 MiB result behind
+    a 112.0 MiB peak — **1.70x and 1.56x the result alone**, but 1.11x and
+    1.02x of ``input + result``.  Those ratios stay near 1 because covering has
+    no coarsen direction that can shrink the result to nothing;
+    :func:`mocs_to_orders` does, and reaches 60x there.  Near 1 is not
+    negligible, though — size a worker off ``input + result``, not the result
+    alone.
 
     Input and output are ragged arrays in arrow list layout: polygon ``i`` is
     ``lats[offsets[i]:offsets[i+1]]`` / ``lons[offsets[i]:offsets[i+1]]``, and

mortie/moc.py

@@ -68,16 +68,17 @@ def moc_to_order(morton, order, max_cells=_FLAT_COVER_WARN_THRESHOLD):
     ``order`` (which coarsen and dedup on densify), where it is a safe over-count
     — so the guard never lets more than ``max_cells`` cells through.
 
-    ``order`` is range-checked here, in the wrapper, for the same reason.  The
-    kernel's densify shift is only defined over 0-29; an out-of-range order
-    reaches it as a Rust panic, surfacing as ``pyo3_runtime.PanicException``,
-    which derives from :class:`BaseException` — so neither ``except ValueError``
-    nor ``except Exception`` catches it.  The budget does not screen it either:
-    the estimate's ``1 << (2 * (order - depth))`` wraps mod 64 in a release
-    build, so for depth-6 input the whole band ``order`` 38-48 estimates *under*
-    the default budget and passes through to the panic.  Refusing with the
-    :class:`ValueError` this contract already promises keeps it catchable by the
-    handlers consumers already have (issue #108).
+    ``order`` is range-checked here, in the wrapper, for the same reason: the
+    refusal must be catchable by the handlers consumers already have (issue
+    #108), and a Rust panic surfaces as ``pyo3_runtime.PanicException``, which
+    derives from :class:`BaseException` — so neither ``except ValueError`` nor
+    ``except Exception`` catches it.  The kernel's densify shift is defined only
+    over 0-29, and past that ``1 << (2 * (order - depth))`` used to wrap mod 64
+    in a release build rather than trap: for depth-6 input the whole band
+    ``order`` 38-48 estimated *under* the default budget and passed straight
+    through to the panic.  That shift now refuses out of range in Rust too and
+    the binding maps it to the same :class:`ValueError` (issue #161), so this
+    check is defence in depth rather than the only defence.
 
     Parameters
     ----------

mortie/tests/test_coverage.py

@@ -683,6 +683,60 @@ def test_order_zero_is_in_range(self):
         dens = mortie.moc_to_order(self.BASE_CELL, 0)
         np.testing.assert_array_equal(dens, self.BASE_CELL)
 
+    def test_kernel_refuses_out_of_range_order_behind_the_wrapper(self):
+        # The wrapper guard above is a fence; the kernel is now correct on its
+        # own (issue #161). Call the bindings directly, past the fence, at the
+        # same orders: `1 << (2 * (order - depth))` used to wrap mod 64 -- a
+        # depth-6 word estimated 1 cell at order 38 (under budget, straight
+        # through to the densify's PanicException) and 1125899906842624 at 255.
+        # Both entry points must raise a ValueError that `except Exception`
+        # catches, since PanicException derives from BaseException and does not.
+        from mortie import _rustie
+
+        depth6 = np.atleast_1d(mortie.norm2mort(0, 0, 6)).astype(np.uint64)
+        for order in (30, 38, 44, 48, 70, 255):
+            for call in (_rustie.rust_moc_to_order_count,
+                         _rustie.rust_moc_to_order):
+                try:
+                    call(depth6, order)
+                except Exception as exc:  # must not escape as PanicException
+                    assert isinstance(exc, ValueError), type(exc)
+                    assert f"between 0 and 29, got {order}" in str(exc)
+                else:
+                    raise AssertionError(
+                        f"{call.__name__} order={order} did not raise")
+
+    def test_malformed_word_is_a_catchable_valueerror(self):
+        # The word half of the same contract (issue #161 review): the empty
+        # word 0 panics in `mort2nested`, and neither binding converted it --
+        # so `moc_to_order` on it raised PanicException on the default budget
+        # (the estimate decodes first) and again with max_cells=None (the
+        # densify decodes). Both are now captured into a plain ValueError.
+        from mortie import _rustie
+
+        bad = np.zeros(1, dtype=np.uint64)
+        for max_cells in (1 << 20, None):
+            try:
+                mortie.moc_to_order(bad, 5, max_cells=max_cells)
+            except Exception as exc:  # must not escape as PanicException
+                assert isinstance(exc, ValueError), type(exc)
+                assert "Morton index cannot be zero" in str(exc)
+            else:
+                raise AssertionError(f"max_cells={max_cells} did not raise")
+        for call in (_rustie.rust_moc_to_order_count, _rustie.rust_moc_to_order):
+            with pytest.raises(ValueError, match="Morton index cannot be zero"):
+                call(bad, 5)
+
+    def test_kernel_estimate_is_exact_at_the_boundary_order(self):
+        # The refusal is `order > 29`, inclusive at 29 -- the deepest order the
+        # packed word represents, not an out-of-range one. A depth-28 word
+        # densifies to its 4 children, and the estimate agrees exactly.
+        from mortie import _rustie
+
+        deep = np.atleast_1d(mortie.norm2mort(0, 0, 28)).astype(np.uint64)
+        assert int(_rustie.rust_moc_to_order_count(deep, 29)) == 4
+        assert len(np.asarray(_rustie.rust_moc_to_order(deep, 29))) == 4
+
 
 class TestCoverageHighOrder:
     """Order 19–29 coverage (issue #60).

mortie/tests/test_moc_batch.py

@@ -394,6 +394,39 @@ def test_budget_refusal_precedes_any_densify():
         mortie.mocs_to_orders(values, [0, 1, 2], 29)
 
 
+def test_malformed_word_is_a_named_value_error_at_both_budgets():
+    """A bad word is the same named ``ValueError`` with the budget on or off.
+
+    The budget estimate decodes every word in the serial pre-pass, so it panics
+    in ``mort2nested`` on the empty word 0 exactly as the densify kernel does.
+    That pass ran outside the kernel's panic capture, so the *default*
+    ``max_cells`` turned the refusal into a ``pyo3_runtime.PanicException`` --
+    ``BaseException``-derived, missed by ``except ValueError`` *and* by
+    ``except Exception`` -- while ``max_cells=None`` gave the documented
+    ``ValueError``.  Same input, opposite exception contract, chosen by a
+    keyword default (issue #161).
+    """
+    from mortie import coverage
+
+    values = np.concatenate([np.zeros(1, np.uint64), _word(600, 6)])
+    for max_cells in (coverage._FLAT_COVER_WARN_THRESHOLD, None):
+        with pytest.raises(ValueError, match=r"moc 0: Morton index cannot be zero"):
+            mortie.mocs_to_orders(values, [0, 2], 8, max_cells=max_cells)
+    # The default is that same budget, and must behave identically.
+    with pytest.raises(ValueError, match=r"moc 0: Morton index cannot be zero"):
+        mortie.mocs_to_orders(values, [0, 2], 8)
+    # A plain ``except Exception`` must see it -- the PanicException lesson.
+    caught = None
+    try:
+        mortie.mocs_to_orders(values, [0, 2], 8)
+    except Exception as exc:
+        caught = exc
+    assert isinstance(caught, ValueError), caught
+    # The lowest-index rule holds when the bad word is not MOC 0.
+    with pytest.raises(ValueError, match=r"moc 1: Morton index cannot be zero"):
+        mortie.mocs_to_orders(values[::-1].copy(), [0, 1, 2], 8)
+
+
 # ---------------------------------------------------------------------------
 # GIL release
 # ---------------------------------------------------------------------------

src_rust/src/coverage.rs

@@ -85,7 +85,7 @@ pub fn polygon_to_morton_coverage(
     normalize: bool,
 ) -> Vec<u64> {
     let moc = polygon_descend(lats, lons, order, None, normalize);
-    crate::moc::to_order(&moc, order)
+    crate::moc::to_order(&moc, order).expect("polygon_descend asserted order 1-29")
 }
 
 /// Compute polygon coverage as a compact, normalized Multi-Order Coverage map:
@@ -157,7 +157,7 @@ pub fn multipolygon_to_morton_coverage(
     validate_multi(lats, lons, order);
     let (rings, refs) = build_rings(lats, lons, normalize);
     let moc = nodes_to_morton(&descend_parallel(&rings, &refs, order, None));
-    crate::moc::to_order(&moc, order)
+    crate::moc::to_order(&moc, order).expect("validate_multi asserted order 1-29")
 }
 
 /// MOC coverage of a ring-set, with optional `tolerance` or `max_cells` stop.

src_rust/src/coverage/batch.rs

@@ -15,8 +15,23 @@
 //!
 //! Polygons are covered a chunk at a time and each chunk is copied into the
 //! ragged output as it lands, so the whole batch's per-polygon covers never
-//! coexist with the concatenated result — peak ≈ result + one chunk, against
-//! the ~2.5x of a cover-everything-then-concatenate pass.
+//! coexist with the concatenated result — against the ~2.5x of a
+//! cover-everything-then-concatenate pass.
+//!
+//! Peak is therefore **input copy + result + one chunk** (issue #162).  The
+//! input term is the pyfunction's `to_vec()`: a `&[f64]` borrowed from numpy
+//! cannot cross `py.allow_threads` (it fails the `Ungil` bound), so `lats`,
+//! `lons` and `offsets` are each copied whole before the GIL is released and
+//! stay resident for the call.  Nothing short of giving up the GIL release
+//! removes it — it is a floor, not an inefficiency.  Measured on a cold call
+//! over synthetic ~1° footprints at order 8: 100k is 6.9 MiB of input and a
+//! 12.9 MiB result behind a 21.9 MiB peak, and 555,867 (catalog scale) is
+//! 38.2 MiB of input and a 71.6 MiB result behind a 112.0 MiB peak — 1.70x and
+//! 1.56x the *result alone*, but 1.11x and 1.02x of `input + result`.  Those
+//! ratios stay near 1 because this path has no coarsen direction that can
+//! shrink the result to nothing — [`crate::moc::batch`]'s densify does, and
+//! reaches 60x there — but near 1 is not the same as negligible, and the copy
+//! is the difference between the two models above.
 //!
 //! # Error posture
 //!
@@ -42,7 +57,9 @@ use super::{polygon_to_morton_moc, polygon_to_morton_moc_budget, polygon_to_mort
 /// output buffer is filled (review of issue #153).  Large enough that the
 /// per-chunk fork-join is noise against the covering work: a sweep of
 /// 1024/2048/8192 on 100k order-8 footprints put 2048 at the pre-chunking
-/// throughput with the peak still ~1.1x the result.
+/// throughput with the peak still ~1.1x of `input + result` (the module
+/// header's model; that sweep quoted it against the result alone, which omits
+/// the input copy — issue #162).
 pub(crate) const CHUNK: usize = 2048;
 
 /// A batch MOC build: ragged `values` / `offsets` (arrow list layout) plus the

src_rust/src/coverage/tests.rs

@@ -255,7 +255,7 @@ fn test_moc_is_compact_and_densifies_to_flat() {
         "interior should collapse to coarse cells"
     );
     assert_eq!(
-        crate::moc::to_order(&moc, 8),
+        crate::moc::to_order(&moc, 8).unwrap(),
         flat,
         "MOC must densify to flat"
     );

src_rust/src/dissolve.rs

@@ -105,7 +105,7 @@ fn debug_assert_cover_on_left(rings: &[Vec<Vec3>], morton: &[u64]) {
     let depths: Vec<u8> = morton.iter().map(|&w| mort2nested(w).1).collect();
     let max_depth = *depths.iter().max().unwrap();
     let flat: Vec<u64> = if depths.iter().any(|&d| d != max_depth) {
-        moc::to_order(morton, max_depth)
+        moc::to_order(morton, max_depth).expect("max_depth decoded from words is <= 29")
     } else {
         morton.to_vec()
     };
@@ -165,7 +165,7 @@ fn survivor_edges(morton: &[u64], step: u32) -> (Vec<(u32, u32)>, Vec<Vec3>) {
     let max_depth = *depths.iter().max().unwrap();
     let min_depth = *depths.iter().min().unwrap();
     let flat: Vec<u64> = if min_depth != max_depth {
-        moc::to_order(morton, max_depth)
+        moc::to_order(morton, max_depth).expect("max_depth decoded from words is <= 29")
     } else {
         morton.to_vec()
     };

src_rust/src/dissolve/tests.rs

@@ -75,7 +75,7 @@ fn assert_point_sampled(cover: &[u64], out: &ClassifiedRings, sample_order: u8)
     let depths: Vec<u8> = cover.iter().map(|&w| mort2nested(w).1).collect();
     let max_depth = *depths.iter().max().unwrap();
     let flat: Vec<u64> = if depths.iter().any(|&d| d != max_depth) {
-        moc::to_order(cover, max_depth)
+        moc::to_order(cover, max_depth).unwrap()
     } else {
         cover.to_vec()
     };

src_rust/src/lib.rs

@@ -1013,6 +1013,11 @@ fn rust_moc_normalize(py: Python<'_>, morton: PyReadonlyArray1<u64>) -> PyResult
 }
 
 /// Densify a (mixed-order) morton set to a flat list at `order`.
+///
+/// `order` above 29 raises `ValueError` — the densify shift is undefined there
+/// and used to wrap mod 64 into a `PanicException` (issue #161).  A malformed
+/// *word* is a `ValueError` too: the decode panic in `mort2nested` is captured
+/// here, the way `rust_mort2nested` captures its own.
 #[pyfunction]
 #[pyo3(signature = (morton, order))]
 fn rust_moc_to_order(
@@ -1021,12 +1026,22 @@ fn rust_moc_to_order(
     order: u8,
 ) -> PyResult<PyObject> {
     let data = morton.to_vec()?;
-    let densified = py.allow_threads(|| moc::to_order(&data, order));
+    let densified = py
+        .allow_threads(|| std::panic::catch_unwind(|| moc::to_order(&data, order)))
+        .map_err(|e| PyValueError::new_err(panic_msg(e, "moc_to_order panicked")))?
+        .map_err(PyValueError::new_err)?;
     Ok(densified.into_pyarray_bound(py).into_any().unbind())
 }
 
 /// Exact flat cell count `rust_moc_to_order` would produce at `order`, computed
 /// from the compact MOC without materializing the flat list (issue #80).
+///
+/// Shares `rust_moc_to_order`'s `order` domain and raises the same `ValueError`
+/// past it, so no `order` can make the guard's estimate a fabricated one
+/// (issue #161).  A malformed *word* raises `ValueError` here as well: the
+/// estimate decodes every word, so it runs under the same panic capture the
+/// densify does — the guard cannot turn a bad word into a `PanicException`
+/// that `except ValueError` misses.
 #[pyfunction]
 #[pyo3(signature = (morton, order))]
 fn rust_moc_to_order_count(
@@ -1035,7 +1050,9 @@ fn rust_moc_to_order_count(
     order: u8,
 ) -> PyResult<u64> {
     let data = morton.to_vec()?;
-    Ok(py.allow_threads(|| moc::to_order_count(&data, order)))
+    py.allow_threads(|| std::panic::catch_unwind(|| moc::to_order_count(&data, order)))
+        .map_err(|e| PyValueError::new_err(panic_msg(e, "moc_to_order_count panicked")))?
+        .map_err(PyValueError::new_err)
 }
 
 /// Densify many (mixed-order) MOCs to a flat `order` in one call (issue #156).