diff --git a/.gitignore b/.gitignore
index 427afc934d..494dec71aa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -670,6 +670,7 @@ backend/backend/settings/*
!backend/backend/settings/base.py
!backend/backend/settings/dev.py
!backend/backend/settings/test.py
+!backend/backend/settings/test_base.py
# Local Dependencies for docker testing
unstract/unstract-sdk/
diff --git a/backend/backend/settings/test.py b/backend/backend/settings/test.py
index 04fa607682..192fefa4c3 100644
--- a/backend/backend/settings/test.py
+++ b/backend/backend/settings/test.py
@@ -1,16 +1,8 @@
-from backend.settings.base import * # noqa: F401, F403
-
-DEBUG = True
+"""OSS test settings: the production base plus the shared test-only deltas.
-# Django's default PBKDF2 hasher is deliberately slow; suites that seed several
-# users per test spend most of their time there. Test fixtures need speed, not
-# resistance to offline cracking.
-PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+The deltas live in `test_base` rather than here because `copy_cloud_deps`
+replaces this file on a cloud build.
+"""
-# The organization-scoped MCP server ships disabled (see
-# MCP_PLATFORM_SERVER_ENABLED in base.py), but its tests must still exercise
-# it: several drive requests through the full URL stack precisely so the auth
-# middleware runs, and an unmounted route would make them 404 — turning a suite
-# that checks a credential is *rejected* into one that passes because nothing
-# is there. Shipping-off and untested are different things.
-MCP_PLATFORM_SERVER_ENABLED = True
+from backend.settings.base import * # noqa: F401, F403
+from backend.settings.test_base import * # noqa: F401, F403
diff --git a/backend/backend/settings/test_base.py b/backend/backend/settings/test_base.py
new file mode 100644
index 0000000000..76af7b3323
--- /dev/null
+++ b/backend/backend/settings/test_base.py
@@ -0,0 +1,19 @@
+"""Test-only setting deltas, shared by the OSS and cloud test settings.
+
+Imports nothing on purpose: `copy_cloud_deps` replaces the OSS `settings/test.py`
+on a cloud build, so deltas kept only there are lost, and star-importing a base
+module here would re-export its names over whatever the importer derived.
+"""
+
+DEBUG = True
+
+# PBKDF2 is deliberately slow, and suites seeding users per test pay it repeatedly.
+PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+
+# Prod enforces HTTPS-only cookies; tests run over plain HTTP.
+SESSION_COOKIE_SECURE = False
+CSRF_COOKIE_SECURE = False
+
+# Ships disabled, but its tests drive the full URL stack to exercise the auth
+# middleware — an unmounted route would 404 and pass them vacuously.
+MCP_PLATFORM_SERVER_ENABLED = True
diff --git a/tests/rig/cli.py b/tests/rig/cli.py
index f93fe622f4..caeafae07a 100644
--- a/tests/rig/cli.py
+++ b/tests/rig/cli.py
@@ -371,8 +371,19 @@ def cmd_report(args: argparse.Namespace) -> int:
print(f"[rig] {exc}", file=sys.stderr)
baseline = None
baseline_corrupt = True
+ # A skipped tier emits no junit, so scoping to the groups that reported keeps
+ # its paths as gaps rather than regressions — nothing regressed, it never ran.
+ # A group that ran and went red still reports, so real regressions survive.
+ # `optional` groups are excluded: they are documented as non-blocking, and
+ # leaving them in scope would let a red one gate the build through a
+ # regression instead.
+ scope_groups = [r.name for r in group_results if not manifest.get(r.name).optional]
statuses = cp.evaluate(
- registry, groups_run_green=green, baseline=baseline, marker_proven=proven
+ registry,
+ groups_run_green=green,
+ baseline=baseline,
+ scope_groups=scope_groups,
+ marker_proven=proven,
)
write_summary(
reports_dir=reports_dir,
@@ -388,6 +399,42 @@ def cmd_report(args: argparse.Namespace) -> int:
f"(not in tests/critical_paths.yaml)",
file=sys.stderr,
)
+ # cmd_run gates on the regressions it can see, but only within its own tier.
+ # This is the only cross-tier evaluation, so it is the only place a regression
+ # spanning tiers can be gated on.
+ # A covering group that ran green without attesting means its marked test
+ # was skipped or unmarked; a group that went red means the test failed. The
+ # remedies differ, so the two are reported apart rather than as one count.
+ unproven: list[cp.CriticalPathStatus] = []
+ uncovered: list[cp.CriticalPathStatus] = []
+ for s in statuses:
+ if s.state != "regression":
+ continue
+ target = unproven if any(g in green for g in s.path.covered_by) else uncovered
+ target.append(s)
+ regressions = unproven + uncovered
+ if uncovered:
+ ids = ", ".join(s.path.id for s in uncovered)
+ print(
+ f"\n[rig] ❌ {len(uncovered)} critical-path regression(s) — no covering "
+ f"group ran green: {ids}",
+ file=sys.stderr,
+ )
+ if unproven:
+ ids = ", ".join(s.path.id for s in unproven)
+ print(
+ f"\n[rig] ❌ {len(unproven)} critical-path regression(s) — covering group "
+ f"ran green but no passing @pytest.mark.critical_path test attested "
+ f"them (skipped or unmarked?): {ids}",
+ file=sys.stderr,
+ )
+ if regressions:
+ print(
+ "[rig] to accept a deliberate removal, drop or re-point the path in "
+ "tests/critical_paths.yaml in the same PR — the baseline is keyed off "
+ "that registry.",
+ file=sys.stderr,
+ )
if args.update_baseline:
red = [
r.name
@@ -404,7 +451,9 @@ def cmd_report(args: argparse.Namespace) -> int:
return 1
cp.merge_into_baseline(statuses, baseline_path)
print(f"[rig] merged into baseline: {baseline_path}")
- return 1 if unknown_marker_ids else 0
+ # A corrupt baseline makes "regression" unreachable, so the gate above would
+ # pass vacuously; it has to fail on its own.
+ return 1 if unknown_marker_ids or regressions or baseline_corrupt else 0
def cmd_run(args: argparse.Namespace) -> int:
diff --git a/tests/rig/critical_paths.py b/tests/rig/critical_paths.py
index e9cb3ccbbb..3b6a72c814 100644
--- a/tests/rig/critical_paths.py
+++ b/tests/rig/critical_paths.py
@@ -226,35 +226,56 @@ def merge_into_baseline(statuses: list[CriticalPathStatus], destination: Path) -
treated as empty: silently dropping previously-covered paths would erase
the other tier's contribution and turn the next build into a regression
festival. CI should delete the cache and retry on this exception.
+
+ Ids absent from the registry are pruned. The union alone never forgets, so a
+ deliberately retired path would otherwise sit in the cache forever; pruning
+ makes editing ``critical_paths.yaml`` the way to accept a removal.
"""
existing: set[str] = set()
if destination.exists():
try:
- parsed = json.loads(destination.read_text())
- existing = set(parsed.get("covered_paths") or [])
- except (json.JSONDecodeError, OSError) as exc:
+ existing = set(_read_baseline(destination)["covered_paths"])
+ except (OSError, ValueError) as exc:
raise BaselineCorruptError(
f"refusing to merge into corrupt baseline {destination}: {exc}. "
"Delete the cache entry and re-run."
) from exc
fresh = {s.path.id for s in statuses if s.state == "covered"}
- payload = {"covered_paths": sorted(existing | fresh)}
+ known = {s.path.id for s in statuses}
+ payload = {"covered_paths": sorted((existing | fresh) & known)}
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(payload, indent=2))
+def _read_baseline(source: Path) -> dict[str, Any]:
+ """Parse a baseline file and check its shape.
+
+ Well-formed JSON of the wrong shape is as unusable as a truncated file, and
+ would otherwise surface as an ``AttributeError``/``TypeError`` from whichever
+ caller touched it first. Raises ``OSError`` or ``ValueError``
+ (``JSONDecodeError`` is one) for callers to translate.
+ """
+ parsed = json.loads(source.read_text())
+ if not isinstance(parsed, dict):
+ raise ValueError(f"expected a JSON object, got {type(parsed).__name__}")
+ covered = parsed.get("covered_paths") or []
+ if not isinstance(covered, list) or not all(isinstance(p, str) for p in covered):
+ raise ValueError("'covered_paths' must be a list of strings")
+ return {**parsed, "covered_paths": covered}
+
+
def load_baseline(source: Path) -> dict[str, Any] | None:
"""Load the cached baseline.
Returns None if the file doesn't exist (first build / fresh cache).
- Raises :class:`BaselineCorruptError` if the file exists but is unreadable
- or unparseable — see :func:`merge_into_baseline` for the rationale.
+ Raises :class:`BaselineCorruptError` if the file exists but is unreadable,
+ unparseable, or misshapen — see :func:`merge_into_baseline` for the rationale.
"""
if not source.exists():
return None
try:
- return json.loads(source.read_text())
- except (json.JSONDecodeError, OSError) as exc:
+ return _read_baseline(source)
+ except (OSError, ValueError) as exc:
raise BaselineCorruptError(
f"baseline at {source} is unreadable: {exc}. "
"Delete the cache entry and re-run."
diff --git a/tests/rig/reporting.py b/tests/rig/reporting.py
index 4c8ed8eda4..897b9f895e 100644
--- a/tests/rig/reporting.py
+++ b/tests/rig/reporting.py
@@ -318,7 +318,15 @@ def _render_markdown(
if critical_statuses:
regressions = [s for s in critical_statuses if s.state == "regression"]
- gaps = [s for s in critical_statuses if s.state == "gap"]
+ # A path whose covering groups exist but reported nothing is not "not yet
+ # covered" — listing it as such reads as missing tests.
+ unexercised: list[CriticalPathStatus] = []
+ gaps: list[CriticalPathStatus] = []
+ for s in critical_statuses:
+ if s.state != "gap":
+ continue
+ target = unexercised if not s.in_scope and s.path.covered_by else gaps
+ target.append(s)
covered = [s for s in critical_statuses if s.state == "covered"]
lines.append("## Critical paths")
@@ -339,6 +347,21 @@ def _render_markdown(
f"(declared coverage: {covers})"
)
lines.append("")
+ if unexercised:
+ lines.append(
+ "💤 Covered, but not exercised in this "
+ "build
"
+ )
+ lines.append("")
+ for s in unexercised:
+ covers = ", ".join(s.path.covered_by)
+ lines.append(
+ f"- **{s.path.id}** — {s.path.description} "
+ f"(covered by {covers}; no result reported in this build)"
+ )
+ lines.append("")
+ lines.append("✅ Covered critical paths
")
lines.append("")
diff --git a/tests/rig/tests/test_cli.py b/tests/rig/tests/test_cli.py
index 17138bf6dc..53495a24db 100644
--- a/tests/rig/tests/test_cli.py
+++ b/tests/rig/tests/test_cli.py
@@ -955,3 +955,162 @@ def test_resolve_configfile_ignores_commented_section(tmp_path: Path) -> None:
child.mkdir()
assert cli._resolve_pytest_configfile(child) is None
+
+
+def _drive_cmd_report(
+ tmp_path: Path,
+ monkeypatch,
+ *,
+ reporting_groups: dict[str, int],
+ baseline_blob: str = '{"covered_paths": ["int-path"]}',
+ optional: bool = False,
+) -> tuple[int, dict[str, str]]:
+ """Drive ``cmd_report`` over one integration group covering ``int-path``.
+
+ ``reporting_groups`` maps group name -> failure count for the groups that
+ emitted junit this build. A group absent from it emitted nothing, standing
+ in for a tier the CI path filter skipped. ``baseline_blob`` is written as the
+ cached baseline verbatim, so a caller can hand over an unparseable one.
+ Returns the exit code and each path's final state.
+ """
+ from tests.rig.reporting import GroupResult
+
+ test_dir = Path(__file__).parent
+ manifest_yaml = (
+ "version: 1\n"
+ "groups:\n"
+ " int-g:\n"
+ " tier: integration\n"
+ f" workdir: {test_dir}\n"
+ " paths: [.]\n"
+ f" optional: {str(optional).lower()}\n"
+ )
+ (tmp_path / "groups.yaml").write_text(manifest_yaml)
+ (tmp_path / "critical_paths.yaml").write_text(
+ "version: 1\n"
+ "paths:\n"
+ " - id: int-path\n"
+ " description: covered only by the integration tier\n"
+ " covered_by: [int-g]\n"
+ )
+
+ import tests.rig.cli as cli_mod
+ import tests.rig.critical_paths as cp_mod
+ import tests.rig.groups as groups_mod
+
+ monkeypatch.setattr(groups_mod, "DEFAULT_MANIFEST", tmp_path / "groups.yaml")
+ monkeypatch.setattr(cp_mod, "DEFAULT_REGISTRY", tmp_path / "critical_paths.yaml")
+ # Coverage combining needs real .coverage files and is not under test here.
+ monkeypatch.setattr(cli_mod, "combine_and_report", lambda reports_dir: None)
+
+ def fake_parse_junit(name, tier, reports_dir):
+ if name not in reporting_groups:
+ return None
+ failed = reporting_groups[name]
+ return GroupResult(
+ name=name,
+ tier=tier,
+ exit_code=1 if failed else 0,
+ passed=0 if failed else 1,
+ failed=failed,
+ errors=0,
+ skipped=0,
+ duration_seconds=0.01,
+ )
+
+ monkeypatch.setattr(cli_mod, "parse_junit", fake_parse_junit)
+
+ states: dict[str, str] = {}
+ real_evaluate = cp_mod.evaluate
+
+ def spy_evaluate(*args, **kwargs):
+ statuses = real_evaluate(*args, **kwargs)
+ states.update({s.path.id: s.state for s in statuses})
+ return statuses
+
+ monkeypatch.setattr(cli_mod.cp, "evaluate", spy_evaluate)
+
+ reports_dir = tmp_path / "reports"
+ reports_dir.mkdir()
+ # Baseline says the path was green on main, which is what makes the
+ # not-covered-now decision a regression-or-gap question at all.
+ (reports_dir / "previous-summary.json").write_text(baseline_blob)
+
+ args = cli_mod._build_parser().parse_args(
+ [
+ "report",
+ "combine",
+ "--reports-dir",
+ str(reports_dir),
+ "--baseline",
+ str(reports_dir / "previous-summary.json"),
+ ]
+ )
+ return cli_mod.cmd_report(args), states
+
+
+def test_cmd_report_skipped_tier_is_a_gap_not_a_regression(
+ tmp_path: Path, monkeypatch
+) -> None:
+ """A path filter can skip a whole tier, whose groups then emit no junit. Its
+ paths must degrade to ``gap``: nothing regressed, the tier never ran.
+ """
+ exit_code, states = _drive_cmd_report(tmp_path, monkeypatch, reporting_groups={})
+
+ assert states["int-path"] == "gap", (
+ "a path whose tier did not run must not be called a regression; "
+ f"got {states['int-path']}"
+ )
+ assert exit_code == 0
+
+
+def test_cmd_report_gates_on_a_real_regression(tmp_path: Path, monkeypatch) -> None:
+ """The covering group ran and went red, so the path really did regress.
+ Being the only cross-tier evaluation, ``cmd_report`` has to gate on it rather
+ than just render it into the comment.
+ """
+ exit_code, states = _drive_cmd_report(
+ tmp_path, monkeypatch, reporting_groups={"int-g": 1}
+ )
+
+ assert states["int-path"] == "regression"
+ assert exit_code == 1, (
+ "a regression must fail the report job, not just print into the comment"
+ )
+
+
+def test_cmd_report_does_not_gate_on_an_optional_group(
+ tmp_path: Path, monkeypatch
+) -> None:
+ """``optional`` groups are documented as non-blocking, and ``cmd_run`` honours
+ that. A red optional group must not reach the gate here through a regression,
+ or the two commands disagree about the same result.
+ """
+ exit_code, states = _drive_cmd_report(
+ tmp_path, monkeypatch, reporting_groups={"int-g": 1}, optional=True
+ )
+
+ assert states["int-path"] == "gap", (
+ f"an optional group's red result must not read as a regression; "
+ f"got {states['int-path']}"
+ )
+ assert exit_code == 0
+
+
+def test_cmd_report_gates_on_a_corrupt_baseline(tmp_path: Path, monkeypatch) -> None:
+ """An unreadable baseline leaves ``previously_covered`` empty, making the
+ regression state unreachable. Without gating on the corruption itself, the
+ job goes green with regression detection silently off.
+ """
+ exit_code, states = _drive_cmd_report(
+ tmp_path,
+ monkeypatch,
+ reporting_groups={"int-g": 1},
+ baseline_blob='{"covered_paths": ["int-pat',
+ )
+
+ assert states["int-path"] == "gap", (
+ "with no usable baseline the regression state is unreachable, which is "
+ "why the corrupt flag must gate on its own"
+ )
+ assert exit_code == 1
diff --git a/tests/rig/tests/test_critical_paths.py b/tests/rig/tests/test_critical_paths.py
index 214c958d62..903e7c0d5e 100644
--- a/tests/rig/tests/test_critical_paths.py
+++ b/tests/rig/tests/test_critical_paths.py
@@ -59,17 +59,20 @@ def test_regression_when_baseline_covered_but_now_not() -> None:
def test_baseline_merge_unions_with_existing(tmp_path: Path) -> None:
- """Two tier runs in sequence must both contribute to the baseline."""
+ """Two tier runs in sequence must both contribute to the baseline.
+
+ Every invocation reads the same registry and differs only in which groups
+ ran, so each one sees the other tier's paths as uncovered — the union is
+ what stops the second run erasing the first.
+ """
baseline = tmp_path / "previous-summary.json"
- registry_a = _registry(("p1", ("g1",)))
- statuses_a = evaluate(
- registry_a, groups_run_green={"g1"}, baseline=None
- )
+ registry = _registry(("p1", ("g1",)), ("p2", ("g2",)))
+
+ statuses_a = evaluate(registry, groups_run_green={"g1"}, baseline=None)
merge_into_baseline(statuses_a, baseline)
- registry_b = _registry(("p2", ("g2",)))
statuses_b = evaluate(
- registry_b, groups_run_green={"g2"}, baseline=load_baseline(baseline)
+ registry, groups_run_green={"g2"}, baseline=load_baseline(baseline)
)
merge_into_baseline(statuses_b, baseline)
@@ -77,6 +80,29 @@ def test_baseline_merge_unions_with_existing(tmp_path: Path) -> None:
assert sorted(final["covered_paths"]) == ["p1", "p2"]
+def test_baseline_merge_prunes_paths_dropped_from_the_registry(tmp_path: Path) -> None:
+ """Retiring a path must be able to clear it from the baseline.
+
+ The union never forgets on its own, so without pruning a deliberately
+ removed path stays cached forever and there is no in-repo way to accept the
+ removal.
+ """
+ baseline = tmp_path / "previous-summary.json"
+ before = _registry(("p1", ("g1",)), ("p2", ("g2",)))
+ merge_into_baseline(
+ evaluate(before, groups_run_green={"g1", "g2"}, baseline=None), baseline
+ )
+ assert sorted((load_baseline(baseline) or {})["covered_paths"]) == ["p1", "p2"]
+
+ after = _registry(("p1", ("g1",)))
+ merge_into_baseline(
+ evaluate(after, groups_run_green={"g1"}, baseline=load_baseline(baseline)),
+ baseline,
+ )
+
+ assert (load_baseline(baseline) or {})["covered_paths"] == ["p1"]
+
+
def test_by_id_lookup_caches() -> None:
registry = _registry(("p1", ("g1",)), ("p2", ()))
# Two lookups must return identical instances; tests both correctness and
@@ -117,6 +143,27 @@ def test_load_baseline_raises_on_corrupt_file(tmp_path: Path) -> None:
load_baseline(baseline)
+@pytest.mark.parametrize(
+ "blob", ["[]", '{"covered_paths": 1}', '{"covered_paths": ["ok", 2]}']
+)
+def test_baseline_of_the_wrong_shape_is_corrupt(tmp_path: Path, blob: str) -> None:
+ """Parseable JSON of the wrong shape is as unusable as a truncated file.
+
+ Left unchecked it surfaces as an ``AttributeError``/``TypeError`` from
+ whichever caller touched it first, rather than the handled corrupt path.
+ """
+ baseline = tmp_path / "previous-summary.json"
+ baseline.write_text(blob)
+ registry = _registry(("p1", ("g1",)))
+
+ with pytest.raises(BaselineCorruptError):
+ load_baseline(baseline)
+ with pytest.raises(BaselineCorruptError):
+ merge_into_baseline(
+ evaluate(registry, groups_run_green={"g1"}, baseline=None), baseline
+ )
+
+
def test_merge_raises_on_corrupt_existing_baseline(tmp_path: Path) -> None:
"""merge_into_baseline must not silently overwrite a corrupt file — that
would erase the other tier's previously-covered paths.