diff --git a/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts b/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts index dd18b1a9e8..5ab38acbf8 100755 --- a/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts +++ b/LifeOS/install/skills/Evals/Graders/ModelBased/LLMRubric.ts @@ -6,6 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, LLMRubricParams } from '../../Types/index.ts'; import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inference.ts'; +import { CURRENT, EFFORT_MODEL } from '../../../../LIFEOS/TOOLS/models.ts'; import { readFileSync, existsSync } from 'fs'; export class LLMRubricGrader extends BaseGrader { @@ -24,14 +25,22 @@ export class LLMRubricGrader extends BaseGrader { const scale = params.scale ?? '1-5'; // Map model preference to inference level (default to medium/Sonnet) - const levelMap: Record = { - 'claude-haiku-4-5-20251001': 'low', + // Superseded judge IDs, kept so existing eval configs keep resolving. + const legacyLevels: Record = { 'claude-sonnet-4-6': 'medium', 'claude-opus-4-8': 'high', 'claude-opus-4-6': 'high', 'claude-sonnet-4-20250514': 'medium', 'claude-opus-4-20250514': 'high', - 'claude-fable-5': 'max', + }; + // Current lineup is DERIVED from the model registry rather than restated: + // models.ts is the single edit point on a release, so a bump cannot strand + // a current model here and silently demote its judge to the default. + const levelMap: Record = { + ...legacyLevels, + ...Object.fromEntries( + Object.entries(EFFORT_MODEL).map(([level, tier]) => [CURRENT[tier], level as InferenceLevel]), + ), }; const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'medium'; diff --git a/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts b/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts index 4cfe983936..6ec67313d0 100755 --- a/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts +++ b/LifeOS/install/skills/Evals/Graders/ModelBased/NaturalLanguageAssert.ts @@ -6,6 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, NaturalLanguageAssertParams } from '../../Types/index.ts'; import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inference.ts'; +import { CURRENT, EFFORT_MODEL } from '../../../../LIFEOS/TOOLS/models.ts'; export class NaturalLanguageAssertGrader extends BaseGrader { type = 'natural_language_assert' as const; @@ -22,14 +23,22 @@ export class NaturalLanguageAssertGrader extends BaseGrader { } // Map model preference to inference level (default to medium/Sonnet) - const levelMap: Record = { - 'claude-haiku-4-5-20251001': 'low', + // Superseded judge IDs, kept so existing eval configs keep resolving. + const legacyLevels: Record = { 'claude-sonnet-4-6': 'medium', 'claude-opus-4-8': 'high', 'claude-opus-4-6': 'high', 'claude-sonnet-4-20250514': 'medium', 'claude-opus-4-20250514': 'high', - 'claude-fable-5': 'max', + }; + // Current lineup is DERIVED from the model registry rather than restated: + // models.ts is the single edit point on a release, so a bump cannot strand + // a current model here and silently demote its judge to the default. + const levelMap: Record = { + ...legacyLevels, + ...Object.fromEntries( + Object.entries(EFFORT_MODEL).map(([level, tier]) => [CURRENT[tier], level as InferenceLevel]), + ), }; const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'medium'; const requireAll = params.require_all ?? true; diff --git a/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts b/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts index ddea6cc240..2053a8dbd9 100755 --- a/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts +++ b/LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts @@ -6,6 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, PairwiseComparisonParams } from '../../Types/index.ts'; import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inference.ts'; +import { CURRENT, EFFORT_MODEL } from '../../../../LIFEOS/TOOLS/models.ts'; import { readFileSync, existsSync } from 'fs'; export class PairwiseComparisonGrader extends BaseGrader { @@ -29,14 +30,22 @@ export class PairwiseComparisonGrader extends BaseGrader { } // Map model preference to inference level (default to medium/Sonnet) - const levelMap: Record = { - 'claude-haiku-4-5-20251001': 'low', + // Superseded judge IDs, kept so existing eval configs keep resolving. + const legacyLevels: Record = { 'claude-sonnet-4-6': 'medium', 'claude-opus-4-8': 'high', 'claude-opus-4-6': 'high', 'claude-sonnet-4-20250514': 'medium', 'claude-opus-4-20250514': 'high', - 'claude-fable-5': 'max', + }; + // Current lineup is DERIVED from the model registry rather than restated: + // models.ts is the single edit point on a release, so a bump cannot strand + // a current model here and silently demote its judge to the default. + const levelMap: Record = { + ...legacyLevels, + ...Object.fromEntries( + Object.entries(EFFORT_MODEL).map(([level, tier]) => [CURRENT[tier], level as InferenceLevel]), + ), }; const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'medium'; const positionSwap = params.position_swap ?? true;