From 41a74663d5ce487d0fe84ea854c2f8edcc6a7a80 Mon Sep 17 00:00:00 2001 From: elhoim Date: Mon, 27 Jul 2026 09:00:39 +0000 Subject: [PATCH] fix(evals): derive judge model levels from the registry instead of restating them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three model-based graders each hardcoded a judge-model -> inference level map. The lineup moved and the maps did not, so both current top-tier IDs fell through to the default: claude-opus-5 -> medium (should be high) claude-sonnet-5 -> medium (right only by accident) The opus case is the damaging one: an eval asking for the strongest judge silently got Sonnet-tier inference, and nothing surfaced the demotion. Derive the current lineup from EFFORT_MODEL/CURRENT in models.ts, which already calls itself the single edit point on a model release. Superseded IDs stay in a small legacy table so existing eval configs keep resolving. A future model bump now flows through automatically instead of stranding a judge here — the same drift models.ts exists to prevent, and which its own header cites as the cautionary example. Verified: opus-5 -> high, sonnet-5 -> medium, fable-5 -> max, haiku -> low, the five legacy IDs unchanged, unknown and empty still default to medium. All three graders transpile. --- .../skills/Evals/Graders/ModelBased/LLMRubric.ts | 15 ++++++++++++--- .../Graders/ModelBased/NaturalLanguageAssert.ts | 15 ++++++++++++--- .../Graders/ModelBased/PairwiseComparison.ts | 15 ++++++++++++--- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts b/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts index dd18b1a9e8..5ab38acbf8 100755 --- a/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts +++ b/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts @@ -6,6 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, LLMRubricParams } from '../../Types/index.ts'; import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inference.ts'; +import { CURRENT, EFFORT_MODEL } from '../../../../LIFEOS/TOOLS/models.ts'; import { readFileSync, existsSync } from 'fs'; export class LLMRubricGrader extends BaseGrader { @@ -24,14 +25,22 @@ export class LLMRubricGrader extends BaseGrader { const scale = params.scale ?? '1-5'; // Map model preference to inference level (default to medium/Sonnet) - const levelMap: Record = { - 'claude-haiku-4-5-20251001': 'low', + // Superseded judge IDs, kept so existing eval configs keep resolving. + const legacyLevels: Record = { 'claude-sonnet-4-6': 'medium', 'claude-opus-4-8': 'high', 'claude-opus-4-6': 'high', 'claude-sonnet-4-20250514': 'medium', 'claude-opus-4-20250514': 'high', - 'claude-fable-5': 'max', + }; + // Current lineup is DERIVED from the model registry rather than restated: + // models.ts is the single edit point on a release, so a bump cannot strand + // a current model here and silently demote its judge to the default. + const levelMap: Record = { + ...legacyLevels, + ...Object.fromEntries( + Object.entries(EFFORT_MODEL).map(([level, tier]) => [CURRENT[tier], level as InferenceLevel]), + ), }; const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'medium'; diff --git a/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts b/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts index 4cfe983936..6ec67313d0 100755 --- a/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts +++ b/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts @@ -6,6 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, NaturalLanguageAssertParams } from '../../Types/index.ts'; import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inference.ts'; +import { CURRENT, EFFORT_MODEL } from '../../../../LIFEOS/TOOLS/models.ts'; export class NaturalLanguageAssertGrader extends BaseGrader { type = 'natural_language_assert' as const; @@ -22,14 +23,22 @@ export class NaturalLanguageAssertGrader extends BaseGrader { } // Map model preference to inference level (default to medium/Sonnet) - const levelMap: Record = { - 'claude-haiku-4-5-20251001': 'low', + // Superseded judge IDs, kept so existing eval configs keep resolving. + const legacyLevels: Record = { 'claude-sonnet-4-6': 'medium', 'claude-opus-4-8': 'high', 'claude-opus-4-6': 'high', 'claude-sonnet-4-20250514': 'medium', 'claude-opus-4-20250514': 'high', - 'claude-fable-5': 'max', + }; + // Current lineup is DERIVED from the model registry rather than restated: + // models.ts is the single edit point on a release, so a bump cannot strand + // a current model here and silently demote its judge to the default. + const levelMap: Record = { + ...legacyLevels, + ...Object.fromEntries( + Object.entries(EFFORT_MODEL).map(([level, tier]) => [CURRENT[tier], level as InferenceLevel]), + ), }; const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'medium'; const requireAll = params.require_all ?? true; diff --git a/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts b/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts index ddea6cc240..2053a8dbd9 100755 --- a/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts +++ b/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts @@ -6,6 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, PairwiseComparisonParams } from '../../Types/index.ts'; import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inference.ts'; +import { CURRENT, EFFORT_MODEL } from '../../../../LIFEOS/TOOLS/models.ts'; import { readFileSync, existsSync } from 'fs'; export class PairwiseComparisonGrader extends BaseGrader { @@ -29,14 +30,22 @@ export class PairwiseComparisonGrader extends BaseGrader { } // Map model preference to inference level (default to medium/Sonnet) - const levelMap: Record = { - 'claude-haiku-4-5-20251001': 'low', + // Superseded judge IDs, kept so existing eval configs keep resolving. + const legacyLevels: Record = { 'claude-sonnet-4-6': 'medium', 'claude-opus-4-8': 'high', 'claude-opus-4-6': 'high', 'claude-sonnet-4-20250514': 'medium', 'claude-opus-4-20250514': 'high', - 'claude-fable-5': 'max', + }; + // Current lineup is DERIVED from the model registry rather than restated: + // models.ts is the single edit point on a release, so a bump cannot strand + // a current model here and silently demote its judge to the default. + const levelMap: Record = { + ...legacyLevels, + ...Object.fromEntries( + Object.entries(EFFORT_MODEL).map(([level, tier]) => [CURRENT[tier], level as InferenceLevel]), + ), }; const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'medium'; const positionSwap = params.position_swap ?? true;