UN-3638 [MISC] Scope and gate critical-path regressions in the rig report

.gitignore

@@ -670,6 +670,7 @@ backend/backend/settings/*
 !backend/backend/settings/base.py
 !backend/backend/settings/dev.py
 !backend/backend/settings/test.py
+!backend/backend/settings/test_base.py
 
 # Local Dependencies for docker testing
 unstract/unstract-sdk/

backend/backend/settings/test.py

@@ -1,16 +1,8 @@
-from backend.settings.base import *  # noqa: F401, F403
-
-DEBUG = True
+"""OSS test settings: the production base plus the shared test-only deltas.
 
-# Django's default PBKDF2 hasher is deliberately slow; suites that seed several
-# users per test spend most of their time there. Test fixtures need speed, not
-# resistance to offline cracking.
-PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+The deltas live in `test_base` rather than here because `copy_cloud_deps`
+replaces this file on a cloud build.
+"""
 
-# The organization-scoped MCP server ships disabled (see
-# MCP_PLATFORM_SERVER_ENABLED in base.py), but its tests must still exercise
-# it: several drive requests through the full URL stack precisely so the auth
-# middleware runs, and an unmounted route would make them 404 — turning a suite
-# that checks a credential is *rejected* into one that passes because nothing
-# is there. Shipping-off and untested are different things.
-MCP_PLATFORM_SERVER_ENABLED = True
+from backend.settings.base import *  # noqa: F401, F403
+from backend.settings.test_base import *  # noqa: F401, F403

backend/backend/settings/test_base.py

@@ -0,0 +1,19 @@
+"""Test-only setting deltas, shared by the OSS and cloud test settings.
+
+Imports nothing on purpose: `copy_cloud_deps` replaces the OSS `settings/test.py`
+on a cloud build, so deltas kept only there are lost, and star-importing a base
+module here would re-export its names over whatever the importer derived.
+"""
+
+DEBUG = True
+
+# PBKDF2 is deliberately slow, and suites seeding users per test pay it repeatedly.
+PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+
+# Prod enforces HTTPS-only cookies; tests run over plain HTTP.
+SESSION_COOKIE_SECURE = False
+CSRF_COOKIE_SECURE = False
+
+# Ships disabled, but its tests drive the full URL stack to exercise the auth
+# middleware — an unmounted route would 404 and pass them vacuously.
+MCP_PLATFORM_SERVER_ENABLED = True

tests/rig/cli.py

@@ -371,8 +371,19 @@ def cmd_report(args: argparse.Namespace) -> int:
         print(f"[rig] {exc}", file=sys.stderr)
         baseline = None
         baseline_corrupt = True
+    # A skipped tier emits no junit, so scoping to the groups that reported keeps
+    # its paths as gaps rather than regressions — nothing regressed, it never ran.
+    # A group that ran and went red still reports, so real regressions survive.
+    # `optional` groups are excluded: they are documented as non-blocking, and
+    # leaving them in scope would let a red one gate the build through a
+    # regression instead.
+    scope_groups = [r.name for r in group_results if not manifest.get(r.name).optional]
     statuses = cp.evaluate(
-        registry, groups_run_green=green, baseline=baseline, marker_proven=proven
+        registry,
+        groups_run_green=green,
+        baseline=baseline,
+        scope_groups=scope_groups,
+        marker_proven=proven,
     )
     write_summary(
         reports_dir=reports_dir,
@@ -388,6 +399,42 @@ def cmd_report(args: argparse.Namespace) -> int:
             f"(not in tests/critical_paths.yaml)",
             file=sys.stderr,
         )
+    # cmd_run gates on the regressions it can see, but only within its own tier.
+    # This is the only cross-tier evaluation, so it is the only place a regression
+    # spanning tiers can be gated on.
+    # A covering group that ran green without attesting means its marked test
+    # was skipped or unmarked; a group that went red means the test failed. The
+    # remedies differ, so the two are reported apart rather than as one count.
+    unproven: list[cp.CriticalPathStatus] = []
+    uncovered: list[cp.CriticalPathStatus] = []
+    for s in statuses:
+        if s.state != "regression":
+            continue
+        target = unproven if any(g in green for g in s.path.covered_by) else uncovered
+        target.append(s)
+    regressions = unproven + uncovered
+    if uncovered:
+        ids = ", ".join(s.path.id for s in uncovered)
+        print(
+            f"\n[rig] ❌ {len(uncovered)} critical-path regression(s) — no covering "
+            f"group ran green: {ids}",
+            file=sys.stderr,
+        )
+    if unproven:
+        ids = ", ".join(s.path.id for s in unproven)
+        print(
+            f"\n[rig] ❌ {len(unproven)} critical-path regression(s) — covering group "
+            f"ran green but no passing @pytest.mark.critical_path test attested "
+            f"them (skipped or unmarked?): {ids}",
+            file=sys.stderr,
+        )
+    if regressions:
+        print(
+            "[rig] to accept a deliberate removal, drop or re-point the path in "
+            "tests/critical_paths.yaml in the same PR — the baseline is keyed off "
+            "that registry.",
+            file=sys.stderr,
+        )
     if args.update_baseline:
         red = [
             r.name
@@ -404,7 +451,9 @@ def cmd_report(args: argparse.Namespace) -> int:
             return 1
         cp.merge_into_baseline(statuses, baseline_path)
         print(f"[rig] merged into baseline: {baseline_path}")
-    return 1 if unknown_marker_ids else 0
+    # A corrupt baseline makes "regression" unreachable, so the gate above would
+    # pass vacuously; it has to fail on its own.
+    return 1 if unknown_marker_ids or regressions or baseline_corrupt else 0
 
 
 def cmd_run(args: argparse.Namespace) -> int:

tests/rig/critical_paths.py

@@ -226,35 +226,56 @@ def merge_into_baseline(statuses: list[CriticalPathStatus], destination: Path) -
     treated as empty: silently dropping previously-covered paths would erase
     the other tier's contribution and turn the next build into a regression
     festival. CI should delete the cache and retry on this exception.
+
+    Ids absent from the registry are pruned. The union alone never forgets, so a
+    deliberately retired path would otherwise sit in the cache forever; pruning
+    makes editing ``critical_paths.yaml`` the way to accept a removal.
     """
     existing: set[str] = set()
     if destination.exists():
         try:
-            parsed = json.loads(destination.read_text())
-            existing = set(parsed.get("covered_paths") or [])
-        except (json.JSONDecodeError, OSError) as exc:
+            existing = set(_read_baseline(destination)["covered_paths"])
+        except (OSError, ValueError) as exc:
             raise BaselineCorruptError(
                 f"refusing to merge into corrupt baseline {destination}: {exc}. "
                 "Delete the cache entry and re-run."
             ) from exc
     fresh = {s.path.id for s in statuses if s.state == "covered"}
-    payload = {"covered_paths": sorted(existing | fresh)}
+    known = {s.path.id for s in statuses}
+    payload = {"covered_paths": sorted((existing | fresh) & known)}
     destination.parent.mkdir(parents=True, exist_ok=True)
     destination.write_text(json.dumps(payload, indent=2))
 
 
+def _read_baseline(source: Path) -> dict[str, Any]:
+    """Parse a baseline file and check its shape.
+
+    Well-formed JSON of the wrong shape is as unusable as a truncated file, and
+    would otherwise surface as an ``AttributeError``/``TypeError`` from whichever
+    caller touched it first. Raises ``OSError`` or ``ValueError``
+    (``JSONDecodeError`` is one) for callers to translate.
+    """
+    parsed = json.loads(source.read_text())
+    if not isinstance(parsed, dict):
+        raise ValueError(f"expected a JSON object, got {type(parsed).__name__}")
+    covered = parsed.get("covered_paths") or []
+    if not isinstance(covered, list) or not all(isinstance(p, str) for p in covered):
+        raise ValueError("'covered_paths' must be a list of strings")
+    return {**parsed, "covered_paths": covered}
+
+
 def load_baseline(source: Path) -> dict[str, Any] | None:
     """Load the cached baseline.
 
     Returns None if the file doesn't exist (first build / fresh cache).
-    Raises :class:`BaselineCorruptError` if the file exists but is unreadable
-    or unparseable — see :func:`merge_into_baseline` for the rationale.
+    Raises :class:`BaselineCorruptError` if the file exists but is unreadable,
+    unparseable, or misshapen — see :func:`merge_into_baseline` for the rationale.
     """
     if not source.exists():
         return None
     try:
-        return json.loads(source.read_text())
-    except (json.JSONDecodeError, OSError) as exc:
+        return _read_baseline(source)
+    except (OSError, ValueError) as exc:
         raise BaselineCorruptError(
             f"baseline at {source} is unreadable: {exc}. "
             "Delete the cache entry and re-run."

tests/rig/reporting.py

@@ -318,7 +318,15 @@ def _render_markdown(
 
     if critical_statuses:
         regressions = [s for s in critical_statuses if s.state == "regression"]
-        gaps = [s for s in critical_statuses if s.state == "gap"]
+        # A path whose covering groups exist but reported nothing is not "not yet
+        # covered" — listing it as such reads as missing tests.
+        unexercised: list[CriticalPathStatus] = []
+        gaps: list[CriticalPathStatus] = []
+        for s in critical_statuses:
+            if s.state != "gap":
+                continue
+            target = unexercised if not s.in_scope and s.path.covered_by else gaps
+            target.append(s)
         covered = [s for s in critical_statuses if s.state == "covered"]
 
         lines.append("## Critical paths")
@@ -339,6 +347,21 @@ def _render_markdown(
                     f"(declared coverage: {covers})"
                 )
             lines.append("")
+        if unexercised:
+            lines.append(
+                "<details><summary>💤 Covered, but not exercised in this "
+                "build</summary>"
+            )
+            lines.append("")
+            for s in unexercised:
+                covers = ", ".join(s.path.covered_by)
+                lines.append(
+                    f"- **{s.path.id}** — {s.path.description} "
+                    f"(covered by {covers}; no result reported in this build)"
+                )
+            lines.append("")
+            lines.append("</details>")
+            lines.append("")
         if covered:
             lines.append("<details><summary>✅ Covered critical paths</summary>")
             lines.append("")

tests/rig/tests/test_cli.py

@@ -955,3 +955,162 @@ def test_resolve_configfile_ignores_commented_section(tmp_path: Path) -> None:
     child.mkdir()
 
     assert cli._resolve_pytest_configfile(child) is None
+
+
+def _drive_cmd_report(
+    tmp_path: Path,
+    monkeypatch,
+    *,
+    reporting_groups: dict[str, int],
+    baseline_blob: str = '{"covered_paths": ["int-path"]}',
+    optional: bool = False,
+) -> tuple[int, dict[str, str]]:
+    """Drive ``cmd_report`` over one integration group covering ``int-path``.
+
+    ``reporting_groups`` maps group name -> failure count for the groups that
+    emitted junit this build. A group absent from it emitted nothing, standing
+    in for a tier the CI path filter skipped. ``baseline_blob`` is written as the
+    cached baseline verbatim, so a caller can hand over an unparseable one.
+    Returns the exit code and each path's final state.
+    """
+    from tests.rig.reporting import GroupResult
+
+    test_dir = Path(__file__).parent
+    manifest_yaml = (
+        "version: 1\n"
+        "groups:\n"
+        "  int-g:\n"
+        "    tier: integration\n"
+        f"    workdir: {test_dir}\n"
+        "    paths: [.]\n"
+        f"    optional: {str(optional).lower()}\n"
+    )
+    (tmp_path / "groups.yaml").write_text(manifest_yaml)
+    (tmp_path / "critical_paths.yaml").write_text(
+        "version: 1\n"
+        "paths:\n"
+        "  - id: int-path\n"
+        "    description: covered only by the integration tier\n"
+        "    covered_by: [int-g]\n"
+    )
+
+    import tests.rig.cli as cli_mod
+    import tests.rig.critical_paths as cp_mod
+    import tests.rig.groups as groups_mod
+
+    monkeypatch.setattr(groups_mod, "DEFAULT_MANIFEST", tmp_path / "groups.yaml")
+    monkeypatch.setattr(cp_mod, "DEFAULT_REGISTRY", tmp_path / "critical_paths.yaml")
+    # Coverage combining needs real .coverage files and is not under test here.
+    monkeypatch.setattr(cli_mod, "combine_and_report", lambda reports_dir: None)
+
+    def fake_parse_junit(name, tier, reports_dir):
+        if name not in reporting_groups:
+            return None
+        failed = reporting_groups[name]
+        return GroupResult(
+            name=name,
+            tier=tier,
+            exit_code=1 if failed else 0,
+            passed=0 if failed else 1,
+            failed=failed,
+            errors=0,
+            skipped=0,
+            duration_seconds=0.01,
+        )
+
+    monkeypatch.setattr(cli_mod, "parse_junit", fake_parse_junit)
+
+    states: dict[str, str] = {}
+    real_evaluate = cp_mod.evaluate
+
+    def spy_evaluate(*args, **kwargs):
+        statuses = real_evaluate(*args, **kwargs)
+        states.update({s.path.id: s.state for s in statuses})
+        return statuses
+
+    monkeypatch.setattr(cli_mod.cp, "evaluate", spy_evaluate)
+
+    reports_dir = tmp_path / "reports"
+    reports_dir.mkdir()
+    # Baseline says the path was green on main, which is what makes the
+    # not-covered-now decision a regression-or-gap question at all.
+    (reports_dir / "previous-summary.json").write_text(baseline_blob)
+
+    args = cli_mod._build_parser().parse_args(
+        [
+            "report",
+            "combine",
+            "--reports-dir",
+            str(reports_dir),
+            "--baseline",
+            str(reports_dir / "previous-summary.json"),
+        ]
+    )
+    return cli_mod.cmd_report(args), states
+
+
+def test_cmd_report_skipped_tier_is_a_gap_not_a_regression(
+    tmp_path: Path, monkeypatch
+) -> None:
+    """A path filter can skip a whole tier, whose groups then emit no junit. Its
+    paths must degrade to ``gap``: nothing regressed, the tier never ran.
+    """
+    exit_code, states = _drive_cmd_report(tmp_path, monkeypatch, reporting_groups={})
+
+    assert states["int-path"] == "gap", (
+        "a path whose tier did not run must not be called a regression; "
+        f"got {states['int-path']}"
+    )
+    assert exit_code == 0
+
+
+def test_cmd_report_gates_on_a_real_regression(tmp_path: Path, monkeypatch) -> None:
+    """The covering group ran and went red, so the path really did regress.
+    Being the only cross-tier evaluation, ``cmd_report`` has to gate on it rather
+    than just render it into the comment.
+    """
+    exit_code, states = _drive_cmd_report(
+        tmp_path, monkeypatch, reporting_groups={"int-g": 1}
+    )
+
+    assert states["int-path"] == "regression"
+    assert exit_code == 1, (
+        "a regression must fail the report job, not just print into the comment"
+    )
+
+
+def test_cmd_report_does_not_gate_on_an_optional_group(
+    tmp_path: Path, monkeypatch
+) -> None:
+    """``optional`` groups are documented as non-blocking, and ``cmd_run`` honours
+    that. A red optional group must not reach the gate here through a regression,
+    or the two commands disagree about the same result.
+    """
+    exit_code, states = _drive_cmd_report(
+        tmp_path, monkeypatch, reporting_groups={"int-g": 1}, optional=True
+    )
+
+    assert states["int-path"] == "gap", (
+        f"an optional group's red result must not read as a regression; "
+        f"got {states['int-path']}"
+    )
+    assert exit_code == 0
+
+
+def test_cmd_report_gates_on_a_corrupt_baseline(tmp_path: Path, monkeypatch) -> None:
+    """An unreadable baseline leaves ``previously_covered`` empty, making the
+    regression state unreachable. Without gating on the corruption itself, the
+    job goes green with regression detection silently off.
+    """
+    exit_code, states = _drive_cmd_report(
+        tmp_path,
+        monkeypatch,
+        reporting_groups={"int-g": 1},
+        baseline_blob='{"covered_paths": ["int-pat',
+    )
+
+    assert states["int-path"] == "gap", (
+        "with no usable baseline the regression state is unreachable, which is "
+        "why the corrupt flag must gate on its own"
+    )
+    assert exit_code == 1