From b0695c3a2a5fd5d5ce0596a8fc5dea693fd20780 Mon Sep 17 00:00:00 2001 From: aurascoper Date: Mon, 27 Jul 2026 12:34:01 -0500 Subject: [PATCH] test(eval): use an attainable exact Mann-Whitney significance case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No production change. The test asserted a p-value the test's own sample size makes unreachable: with n1 = n2 = 3, the smallest attainable two-sided p is 2/20 = 0.1, so completely disjoint groups still return exactly 0.1 and no correct implementation can clear 0.05. Four per group makes the claim attainable — the exact minimum is 2/70 = 0.029 — and the assertions now pin U and both sample sizes rather than only the p-value. n=3 per group U=0.0 p=0.1000 n=4 per group U=0.0 p=0.0286 Removes the second of three deselections in the python-contracts job (#38). Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 1 - Tests/eval/test_eval_stats.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a45a0d..f56a39f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,5 +119,4 @@ jobs: run: | pytest Tests/eval -v \ --ignore=Tests/eval/test_joint_embedding.py \ - --deselect Tests/eval/test_eval_stats.py::test_mann_whitney_u_disjoint \ --deselect Tests/eval/test_embedding_space.py::test_cka_independent diff --git a/Tests/eval/test_eval_stats.py b/Tests/eval/test_eval_stats.py index ba00050..ac3d6a7 100644 --- a/Tests/eval/test_eval_stats.py +++ b/Tests/eval/test_eval_stats.py @@ -42,9 +42,15 @@ def test_cohens_d_large_nondegenerate_effect(): def test_mann_whitney_u_disjoint(): - result = mann_whitney_u([1, 2, 3], [10, 11, 12]) + # Four per group, not three: with n1 = n2 = 3 the smallest attainable + # two-sided p is 2/20 = 0.1, so no correct implementation can clear 0.05 + # there however cleanly the groups separate. At four per group the exact + # minimum is 2/70 = 0.029. + result = mann_whitney_u([1, 2, 3, 4], [10, 11, 12, 13]) + assert result["u_statistic"] == 0.0 assert result["p_value"] < 0.05 - assert "u_statistic" in result + assert result["n_a"] == 4 + assert result["n_b"] == 4 def test_bonferroni_correct():