diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f56a39f..3f83316 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,19 +104,14 @@ jobs: PYTHONPATH=NeuralComposeEEG/src \ python -m unittest discover -s NeuralComposeEEG/tests -t . --verbose - # pytest, because Tests/eval mixes four harness styles: unittest.TestCase, + # pytest, because Tests/eval mixes harness styles: unittest.TestCase, # bare `def test_*` with a __main__ runner, and bare `def test_*` with no - # runner at all. pytest collects all of them; unittest collects 8 of 62 - # and reports OK for the rest. + # runner at all. pytest collects all of them; `python -m unittest` skips + # most silently and still reports OK, so it cannot gate this directory. # # test_joint_embedding.py imports mlx.core (Apple Silicon only) and would # fail collection here — run it locally. - # - # The remaining deselected cases are pre-existing failures tracked in - # issue #38, reproducing on both NumPy 1.26 and 2.4. They are removed one - # at a time as each is resolved; the rest of their files still run. - name: Test (eval contract suites) run: | pytest Tests/eval -v \ - --ignore=Tests/eval/test_joint_embedding.py \ - --deselect Tests/eval/test_embedding_space.py::test_cka_independent + --ignore=Tests/eval/test_joint_embedding.py diff --git a/Evaluation/scripts/embedding_space_analysis.py b/Evaluation/scripts/embedding_space_analysis.py index 1504ea5..8694c36 100644 --- a/Evaluation/scripts/embedding_space_analysis.py +++ b/Evaluation/scripts/embedding_space_analysis.py @@ -35,6 +35,13 @@ def cka(X, Y): + """Biased linear CKA over centered Gram matrices. + + Null values can be substantially inflated when feature dimension approaches + or exceeds sample count — independent Gaussians at n=50, d=128/256 score + ~0.78, not ~0. Do not compare scores across different n/d regimes without a + matched null or permutation baseline. + """ X = np.array(X, dtype=np.float64) Y = np.array(Y, dtype=np.float64) X = X - X.mean(axis=0) diff --git a/Tests/eval/test_embedding_space.py b/Tests/eval/test_embedding_space.py index 1818692..3a3cff9 100644 --- a/Tests/eval/test_embedding_space.py +++ b/Tests/eval/test_embedding_space.py @@ -20,12 +20,21 @@ def test_cka_identical(): assert abs(score - 1.0) < 1e-3 -def test_cka_independent(): - np.random.seed(42) - X = np.random.randn(50, 128) - Y = np.random.randn(50, 256) - score = cka(X, Y) - assert abs(score) < 0.3 +def test_cka_independent_is_small_when_samples_exceed_dimensions(): + rng = np.random.default_rng(42) + X = rng.standard_normal((500, 16)) + Y = rng.standard_normal((500, 32)) + assert cka(X, Y) < 0.1 + + +def test_biased_cka_null_is_inflated_when_dimensions_exceed_samples(): + # Characterizes the selected biased estimator; NOT evidence that these + # representations are genuinely similar. The previous expectation + # (< 0.3 at n=50, d=128/256) assumed an unbiased estimator or n >> d. + rng = np.random.default_rng(42) + X = rng.standard_normal((50, 128)) + Y = rng.standard_normal((50, 256)) + assert cka(X, Y) > 0.5 def test_svcca_identical():