test(eval): pin biased linear CKA behavior across sample regimes

.github/workflows/ci.yml

@@ -104,19 +104,14 @@ jobs:
           PYTHONPATH=NeuralComposeEEG/src \
             python -m unittest discover -s NeuralComposeEEG/tests -t . --verbose
 
-      # pytest, because Tests/eval mixes four harness styles: unittest.TestCase,
+      # pytest, because Tests/eval mixes harness styles: unittest.TestCase,
       # bare `def test_*` with a __main__ runner, and bare `def test_*` with no
-      # runner at all. pytest collects all of them; unittest collects 8 of 62
-      # and reports OK for the rest.
+      # runner at all. pytest collects all of them; `python -m unittest` skips
+      # most silently and still reports OK, so it cannot gate this directory.
       #
       # test_joint_embedding.py imports mlx.core (Apple Silicon only) and would
       # fail collection here — run it locally.
-      #
-      # The remaining deselected cases are pre-existing failures tracked in
-      # issue #38, reproducing on both NumPy 1.26 and 2.4. They are removed one
-      # at a time as each is resolved; the rest of their files still run.
       - name: Test (eval contract suites)
         run: |
           pytest Tests/eval -v \
-            --ignore=Tests/eval/test_joint_embedding.py \
-            --deselect Tests/eval/test_embedding_space.py::test_cka_independent
+            --ignore=Tests/eval/test_joint_embedding.py

Evaluation/scripts/embedding_space_analysis.py

@@ -35,6 +35,13 @@
 
 
 def cka(X, Y):
+    """Biased linear CKA over centered Gram matrices.
+
+    Null values can be substantially inflated when feature dimension approaches
+    or exceeds sample count — independent Gaussians at n=50, d=128/256 score
+    ~0.78, not ~0. Do not compare scores across different n/d regimes without a
+    matched null or permutation baseline.
+    """
     X = np.array(X, dtype=np.float64)
     Y = np.array(Y, dtype=np.float64)
     X = X - X.mean(axis=0)

Tests/eval/test_embedding_space.py

@@ -20,12 +20,21 @@ def test_cka_identical():
     assert abs(score - 1.0) < 1e-3
 
 
-def test_cka_independent():
-    np.random.seed(42)
-    X = np.random.randn(50, 128)
-    Y = np.random.randn(50, 256)
-    score = cka(X, Y)
-    assert abs(score) < 0.3
+def test_cka_independent_is_small_when_samples_exceed_dimensions():
+    rng = np.random.default_rng(42)
+    X = rng.standard_normal((500, 16))
+    Y = rng.standard_normal((500, 32))
+    assert cka(X, Y) < 0.1
+
+
+def test_biased_cka_null_is_inflated_when_dimensions_exceed_samples():
+    # Characterizes the selected biased estimator; NOT evidence that these
+    # representations are genuinely similar. The previous expectation
+    # (< 0.3 at n=50, d=128/256) assumed an unbiased estimator or n >> d.
+    rng = np.random.default_rng(42)
+    X = rng.standard_normal((50, 128))
+    Y = rng.standard_normal((50, 256))
+    assert cka(X, Y) > 0.5
 
 
 def test_svcca_identical():