From 645eb4f7b241340e4ef2724e862003930dd0f57e Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 2 Sep 2026 22:11:30 +0530 Subject: [PATCH] chore: complete verification tooling integration 1/1 --- .../differential-daemon-contracts.ts | 23 +- apps/desktop/src/pages/Rubrics.tsx | 137 +- apps/desktop/src/pages/Settings.tsx | 124 -- apps/desktop/src/pages/TRex.tsx | 7 +- .../tests/e2e/review-warm-evidence.spec.ts | 3 +- apps/desktop/tests/e2e/settings.spec.ts | 70 +- apps/desktop/tests/e2e/work.spec.ts | 1119 ----------------- .../model-comparison-report-v1.json | 2 +- .../real-pipeline-correctness-v1.json | 4 +- scripts/generate-benchmark-dataset.mjs | 2 +- scripts/map-benchmark-reviews.mjs | 4 +- scripts/merge-landing-docs.mjs | 13 +- .../qualify-codex-accounting-oracle.test.mjs | 199 ++- scripts/run-public-benchmark.mjs | 10 +- scripts/verification-receipts/analyze.mjs | 1 + scripts/verification-receipts/contracts.mjs | 85 +- .../verification-receipts.test.mjs | 129 ++ 17 files changed, 555 insertions(+), 1377 deletions(-) delete mode 100644 apps/desktop/tests/e2e/work.spec.ts diff --git a/apps/desktop/src/lib/warm-verification/differential-daemon-contracts.ts b/apps/desktop/src/lib/warm-verification/differential-daemon-contracts.ts index c13159c9..6a7bc81d 100644 --- a/apps/desktop/src/lib/warm-verification/differential-daemon-contracts.ts +++ b/apps/desktop/src/lib/warm-verification/differential-daemon-contracts.ts @@ -7,6 +7,7 @@ import { VERIFY_CONTRACT_LIMITS, type ContractIssue, type ContractValidation, + type DaemonError, } from './contracts'; import { DIFFERENTIAL_CLASSIFICATIONS, @@ -122,7 +123,8 @@ export type DifferentialDaemonResponse = | { type: 'differential_prepared'; summary: DifferentialPreparedSummary } | { type: 'differential_result'; summary: DifferentialRunSummary } | { type: 'differential_status'; summary: DifferentialStatusSummary } - | { type: 'differential_cleanup'; summary: DifferentialCleanupSummary }; + | { type: 'differential_cleanup'; summary: DifferentialCleanupSummary } + | { type: 'error'; error: DaemonError }; export interface DifferentialDaemonResponseEnvelope { protocol_version: 1; request_id: string; @@ -287,6 +289,25 @@ function validateRequest(value: unknown, issues: ContractIssue[]) { function validateResponse(value: unknown, issues: ContractIssue[]) { const response = object(value, '$.response', issues); if (!response) return; + if (response.type === 'error') { + exactKeys(response, '$.response', ['type', 'error'], issues); + const error = object(response.error, '$.response.error', issues); + if (!error) return; + exactKeys( + error, + '$.response.error', + error.remediation === undefined + ? ['code', 'message', 'retryable'] + : ['code', 'message', 'remediation', 'retryable'], + issues + ); + stringField(error, 'code', '$.response.error', issues, { pattern: ID }); + stringField(error, 'message', '$.response.error', issues); + if (error.remediation !== undefined) + stringField(error, 'remediation', '$.response.error', issues); + boolean(error, 'retryable', '$.response.error', issues); + return; + } exactKeys(response, '$.response', ['type', 'summary'], issues); const rules: Record = { differential_prepared: prepared, diff --git a/apps/desktop/src/pages/Rubrics.tsx b/apps/desktop/src/pages/Rubrics.tsx index 41eaada9..02eaade5 100644 --- a/apps/desktop/src/pages/Rubrics.tsx +++ b/apps/desktop/src/pages/Rubrics.tsx @@ -23,14 +23,16 @@ import { saveReviewConfig, type StandardsPack, } from '@/lib/review-service'; -import { getStandardsPackUsage, isTauriAvailable } from '@/lib/tauri-ipc'; +import { + getRubricSettings, + isTauriAvailable, + type RubricSettingsReceipt, + saveRubricPack, + setActiveRubricPack, +} from '@/lib/tauri-ipc'; function fallbackConfig(): ReviewConfig { return { - gatewayBaseUrl: '', - gatewayApiKey: '', - gatewayModel: 'auto', - reviewTone: 'direct', activeStandardsPack: DEFAULT_STANDARDS_PACKS[0].id, standardsPacks: [], }; @@ -81,47 +83,76 @@ export default function Rubrics({ embedded = false }: { embedded?: boolean }) { const [usage, setUsage] = useState>({}); const [expandedPreview, setExpandedPreview] = useState(null); const [copiedPreview, setCopiedPreview] = useState(null); + const [syncIssue, setSyncIssue] = useState(null); const packs = getStandardsPacks(config); const activePack = getActiveStandardsPack(config); const customRules = config.customRules ?? []; - // Usage is keyed by pack NAME (the value persisted on each review), not id. useEffect(() => { if (!isTauriAvailable()) return; let cancelled = false; - getStandardsPackUsage() - .then((rows) => { + getRubricSettings(loadReviewConfig()) + .then((receipt) => { if (cancelled) return; - const map: Record = {}; - for (const row of rows) { - map[row.standards_pack] = { - reviewCount: row.review_count, - totalFindings: row.total_findings, - }; - } - setUsage(map); + applyCanonicalReceipt(receipt); }) - .catch(() => { - // Non-fatal — packs simply show "no usage yet". + .catch((error) => { + if (cancelled) return; + setSyncIssue(error instanceof Error ? error.message : String(error)); }); return () => { cancelled = true; }; }, []); - function persist(next: ReviewConfig) { + function applyCanonicalReceipt(receipt: RubricSettingsReceipt) { + const standardsPacks = receipt.packs + .filter((pack) => !pack.built_in) + .map(({ id, name, focus, checks }) => ({ id, name, focus, checks })); + const persisted: ReviewConfig = { + customRules: receipt.custom_rules, + standardsPacks, + ...(receipt.active_pack_id ? { activeStandardsPack: receipt.active_pack_id } : {}), + }; + saveReviewConfig(persisted); + setConfig({ + ...persisted, + activeStandardsPack: receipt.active_pack_id ?? DEFAULT_STANDARDS_PACKS[0].id, + }); + setUsage( + Object.fromEntries( + receipt.packs.map((pack) => [ + pack.id, + { reviewCount: pack.review_count, totalFindings: pack.total_findings }, + ]) + ) + ); + setSyncIssue(null); + setSaved(true); + window.setTimeout(() => setSaved(false), 1600); + } + + function persistLocal(next: ReviewConfig) { setConfig(next); saveReviewConfig(next); setSaved(true); window.setTimeout(() => setSaved(false), 1600); } - function selectPack(packId: string) { - persist({ ...config, activeStandardsPack: packId }); + async function selectPack(packId: string) { + if (!isTauriAvailable()) { + persistLocal({ ...config, activeStandardsPack: packId }); + return; + } + try { + applyCanonicalReceipt(await setActiveRubricPack(packId)); + } catch (error) { + setSyncIssue(error instanceof Error ? error.message : String(error)); + } } - function clonePack(source: StandardsPack) { + async function clonePack(source: StandardsPack) { const cloneName = `${source.name} (copy)`; const cloneId = uniquePackId(makePackId(cloneName), packs); const clone: StandardsPack = { @@ -130,12 +161,21 @@ export default function Rubrics({ embedded = false }: { embedded?: boolean }) { focus: source.focus, checks: [...source.checks], }; - persist({ - ...config, - activeStandardsPack: clone.id, - standardsPacks: [...(config.standardsPacks ?? []), clone], - }); - setExpandedPreview(clone.id); + if (!isTauriAvailable()) { + persistLocal({ + ...config, + activeStandardsPack: clone.id, + standardsPacks: [...(config.standardsPacks ?? []), clone], + }); + setExpandedPreview(clone.id); + return; + } + try { + applyCanonicalReceipt(await saveRubricPack(clone)); + setExpandedPreview(clone.id); + } catch (error) { + setSyncIssue(error instanceof Error ? error.message : String(error)); + } } async function copyPreview(pack: StandardsPack) { @@ -149,7 +189,7 @@ export default function Rubrics({ embedded = false }: { embedded?: boolean }) { } } - function addCustomPack() { + async function addCustomPack() { const checks = draftChecks .split('\n') .map((line) => line.trim()) @@ -166,14 +206,26 @@ export default function Rubrics({ embedded = false }: { embedded?: boolean }) { checks, }; - persist({ - ...config, - activeStandardsPack: pack.id, - standardsPacks: [...(config.standardsPacks ?? []), pack], - }); - setDraftName(''); - setDraftFocus(''); - setDraftChecks(''); + if (!isTauriAvailable()) { + persistLocal({ + ...config, + activeStandardsPack: pack.id, + standardsPacks: [...(config.standardsPacks ?? []), pack], + }); + setDraftName(''); + setDraftFocus(''); + setDraftChecks(''); + return; + } + + try { + applyCanonicalReceipt(await saveRubricPack(pack)); + setDraftName(''); + setDraftFocus(''); + setDraftChecks(''); + } catch (error) { + setSyncIssue(error instanceof Error ? error.message : String(error)); + } } return ( @@ -218,6 +270,11 @@ export default function Rubrics({ embedded = false }: { embedded?: boolean }) { Saved )} + {syncIssue && ( +
+ Canonical rubric sync failed. Existing local settings were kept: {syncIssue} +
+ )}
@@ -243,7 +300,7 @@ export default function Rubrics({ embedded = false }: { embedded?: boolean }) {
diff --git a/apps/desktop/src/pages/Settings.tsx b/apps/desktop/src/pages/Settings.tsx index 0d0eca8d..8557710d 100644 --- a/apps/desktop/src/pages/Settings.tsx +++ b/apps/desktop/src/pages/Settings.tsx @@ -6,12 +6,6 @@ import { Button } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; import { Input } from '@/components/ui/input'; import { Separator } from '@/components/ui/separator'; -import { - loadReviewConfig, - PROVIDER_PRESETS, - type ReviewConfig, - saveReviewConfig, -} from '@/lib/review-service'; import type { GitHubAuthStatus, LinearUser, @@ -590,48 +584,6 @@ export default function Settings() { const [claudeCodePath, setClaudeCodePath] = usePref('claude_cli_path', ''); const [codexPath, setCodexPath] = usePref('codex_cli_path', ''); - // AI Provider - const [aiProvider, setAiProvider] = useState('anthropic'); - const [aiBaseUrl, setAiBaseUrl] = useState(''); - const [aiApiKey, setAiApiKey] = useState(''); - const [aiModel, setAiModel] = useState(''); - const [aiConfigSaved, setAiConfigSaved] = useState(false); - - useEffect(() => { - const existing = loadReviewConfig(); - if (existing) { - setAiBaseUrl(existing.gatewayBaseUrl); - setAiApiKey(existing.gatewayApiKey); - setAiModel(existing.gatewayModel); - // Detect provider from URL - if (existing.gatewayBaseUrl.includes('anthropic')) setAiProvider('anthropic'); - else if (existing.gatewayBaseUrl.includes('openai.com')) setAiProvider('openai'); - else if (existing.gatewayBaseUrl.includes('openrouter')) setAiProvider('openrouter'); - else setAiProvider('custom'); - } - }, []); - - function handleProviderChange(provider: string) { - setAiProvider(provider); - setAiConfigSaved(false); - if (provider !== 'custom' && PROVIDER_PRESETS[provider]) { - setAiBaseUrl(PROVIDER_PRESETS[provider].baseUrl); - setAiModel(PROVIDER_PRESETS[provider].model); - } - } - - function handleSaveAiConfig() { - const config: ReviewConfig = { - gatewayBaseUrl: aiBaseUrl, - gatewayApiKey: aiApiKey, - gatewayModel: aiModel, - reviewTone: defaultTone, - }; - saveReviewConfig(config); - setAiConfigSaved(true); - setTimeout(() => setAiConfigSaved(false), 2000); - } - // Notifications const [notifyReviewDone, toggleNotifyReviewDone] = useBoolPref('notify_review_done', true); const [notifyAgentError, toggleNotifyAgentError] = useBoolPref('notify_agent_error', true); @@ -1500,82 +1452,6 @@ export default function Settings() {

- -

- AI Provider -

-
- - - - - { - setAiApiKey(v); - setAiConfigSaved(false); - }} - /> - - {aiProvider === 'custom' && ( - <> - - { - setAiBaseUrl(v); - setAiConfigSaved(false); - }} - /> - - )} - - - - { - setAiModel(v); - setAiConfigSaved(false); - }} - /> - -
- - {!aiApiKey && ( - API key required to run reviews - )} -
-
); diff --git a/apps/desktop/src/pages/TRex.tsx b/apps/desktop/src/pages/TRex.tsx index 11991dc9..8aa2e6ff 100644 --- a/apps/desktop/src/pages/TRex.tsx +++ b/apps/desktop/src/pages/TRex.tsx @@ -408,7 +408,7 @@ export default function TRex() { { @@ -439,7 +439,10 @@ export default function TRex() { onCleanup={handleWarmCleanup} /> - + diff --git a/apps/desktop/tests/e2e/review-warm-evidence.spec.ts b/apps/desktop/tests/e2e/review-warm-evidence.spec.ts index 3bb2e647..0cc4a3c0 100644 --- a/apps/desktop/tests/e2e/review-warm-evidence.spec.ts +++ b/apps/desktop/tests/e2e/review-warm-evidence.spec.ts @@ -412,7 +412,8 @@ test('Review presents deterministic coverage and rejected candidate counts', asy await expect(decision.getByRole('link', { name: 'Runtime evidence' })).toBeVisible(); }); -test('Review shows readiness for an external review agent', async ({ page }) => { +test('Review shows readiness for an external review agent', async ({ page, context }) => { + await context.grantPermissions(['clipboard-read', 'clipboard-write']); await installReviewMock(page, false); await navigateTo(page, '/review'); await waitForNoSpinners(page); diff --git a/apps/desktop/tests/e2e/settings.spec.ts b/apps/desktop/tests/e2e/settings.spec.ts index a5d350f5..a9f5648a 100644 --- a/apps/desktop/tests/e2e/settings.spec.ts +++ b/apps/desktop/tests/e2e/settings.spec.ts @@ -15,67 +15,10 @@ test.describe('Settings page', () => { consoleErrors.assertNoErrors(); }); - // ─── General tab ────────────────────────────────────────────────────── - - test('General tab is selected by default and shows AI Provider section', async ({ page }) => { - // "General" should be the active category - await expect(page.locator('text=General').first()).toBeVisible(); - - // AI Provider section heading - await expect(page.getByRole('heading', { name: 'AI Provider' })).toBeVisible(); - }); - - // ─── Provider dropdown ──────────────────────────────────────────────── - - test('Can select AI provider from dropdown', async ({ page }) => { - // The provider dropdown is a