From dbcfd69bd280baea195251e493c6dc836f2d7c42 Mon Sep 17 00:00:00 2001 From: Jeremy McSpadden <211150+jeremymcs@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:57:47 -0500 Subject: [PATCH] feat: add detected models and second-model review --- client/src/lib/agentModelOptions.test.ts | 50 ++++++ client/src/lib/agentModelOptions.ts | 44 +++++ client/src/lib/fullAppQaSurface.test.ts | 13 ++ client/src/pages/settings.tsx | 100 +++++++++-- docs/public/agent-dispatch.md | 4 +- docs/public/configuration.md | 3 +- server/babysitter.test.ts | 211 ++++++++++++++++++++++- server/babysitter.ts | 129 ++++++++++++-- server/defaultConfig.test.ts | 9 + server/defaultConfig.ts | 3 + server/modelDiscovery.test.ts | 67 +++++++ server/modelDiscovery.ts | 152 ++++++++++++++++ server/routes.test.ts | 32 ++++ server/routes.ts | 14 +- server/sqliteStorage.ts | 28 ++- server/storage.test.ts | 6 + shared/schema.ts | 16 ++ 17 files changed, 840 insertions(+), 41 deletions(-) create mode 100644 client/src/lib/agentModelOptions.test.ts create mode 100644 client/src/lib/agentModelOptions.ts create mode 100644 server/modelDiscovery.test.ts create mode 100644 server/modelDiscovery.ts diff --git a/client/src/lib/agentModelOptions.test.ts b/client/src/lib/agentModelOptions.test.ts new file mode 100644 index 0000000..4cb8b86 --- /dev/null +++ b/client/src/lib/agentModelOptions.test.ts @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import type { AgentModelCatalog } from "@shared/schema"; +import { + buildReviewModelOptions, + findReviewModelSelection, +} from "./agentModelOptions"; + +const catalog: AgentModelCatalog = { + codex: [ + { value: "", label: "CLI default" }, + { value: "gpt-5.6-sol", label: "gpt-5.6-sol" }, + { value: "gpt-5.6-luna", label: "gpt-5.6-luna" }, + ], + claude: [ + { value: "", label: "CLI default" }, + { value: "opus", label: "opus" }, + { value: "sonnet", label: "sonnet" }, + ], +}; + +test("buildReviewModelOptions requires an explicit model and excludes the active primary model", () => { + const options = buildReviewModelOptions(catalog, { + agent: "claude", + model: "opus", + }); + + assert.deepEqual( + options.map(({ agent, model }) => [agent, model]), + [ + ["codex", "gpt-5.6-sol"], + ["codex", "gpt-5.6-luna"], + ["claude", "sonnet"], + ], + ); +}); + +test("findReviewModelSelection returns the agent and model represented by the selected option", () => { + const options = buildReviewModelOptions(catalog, { + agent: "codex", + model: "gpt-5.6-sol", + }); + const sonnet = options.find((option) => option.model === "sonnet"); + + assert.deepEqual(findReviewModelSelection(options, sonnet?.value ?? ""), { + agent: "claude", + model: "sonnet", + }); + assert.equal(findReviewModelSelection(options, "unknown"), null); +}); diff --git a/client/src/lib/agentModelOptions.ts b/client/src/lib/agentModelOptions.ts new file mode 100644 index 0000000..9e1e2de --- /dev/null +++ b/client/src/lib/agentModelOptions.ts @@ -0,0 +1,44 @@ +import type { AgentModelCatalog, Config } from "@shared/schema"; + +export type ReviewModelSelection = { + agent: Config["reviewAgent"]; + model: string; +}; + +export type ReviewModelOption = ReviewModelSelection & { + value: string; + label: string; +}; + +function selectionValue(agent: Config["reviewAgent"], model: string): string { + return `${agent}:${encodeURIComponent(model)}`; +} + +export function buildReviewModelOptions( + catalog: AgentModelCatalog, + primary: ReviewModelSelection, +): ReviewModelOption[] { + const optionsFor = (agent: Config["reviewAgent"], label: string) => + catalog[agent] + .filter((option) => option.value.length > 0) + .filter((option) => agent !== primary.agent || option.value !== primary.model) + .map((option) => ({ + agent, + model: option.value, + value: selectionValue(agent, option.value), + label: `${label} · ${option.label}`, + })); + + return [ + ...optionsFor("codex", "Codex"), + ...optionsFor("claude", "Claude"), + ]; +} + +export function findReviewModelSelection( + options: readonly ReviewModelOption[], + value: string, +): ReviewModelSelection | null { + const selected = options.find((option) => option.value === value); + return selected ? { agent: selected.agent, model: selected.model } : null; +} diff --git a/client/src/lib/fullAppQaSurface.test.ts b/client/src/lib/fullAppQaSurface.test.ts index dab0ded..bb1ed0c 100644 --- a/client/src/lib/fullAppQaSurface.test.ts +++ b/client/src/lib/fullAppQaSurface.test.ts @@ -475,6 +475,7 @@ test("settings keeps the QA-tested configuration, token, and runtime controls wi const { sourceFile } = await parseProjectFile("client/src/pages/settings.tsx"); assertHasQueryKey(sourceFile, "settings config query", "/api/config"); + assertHasQueryKey(sourceFile, "detected agent models query", "/api/agent-models"); assertHasQueryKey(sourceFile, "runtime query", "/api/runtime"); assertHasQueryKey(sourceFile, "repo settings query", "/api/repos/settings"); assertHasQueryKey(sourceFile, "GitHub auth status query", "/api/github-auth/status"); @@ -488,6 +489,16 @@ test("settings keeps the QA-tested configuration, token, and runtime controls wi assertHasApiRequest(sourceFile, "manual release mutation", "POST", "/api/repos/release"); assertHasJsxAttribute(sourceFile, "id", "coding agent selector", "settings-coding-agent"); + assertHasExpression( + sourceFile, + "review model selection maps to config fields", + /findReviewModelSelection\(reviewModelOptions,[\s\S]*?updateConfigMutation\.mutate\(\{[\s\S]*?reviewAgent: selection\.agent,[\s\S]*?reviewModel: selection\.model/, + ); + assertHasExpression( + sourceFile, + "review toggle requires a selected model", + /disabled=\{updateConfigMutation\.isPending \|\| !selectedReviewOption\}/, + ); for (const [label, testId] of [ ["add PR input", "input-add-pr"], ["add PR submit", "button-add-pr"], @@ -498,6 +509,8 @@ test("settings keeps the QA-tested configuration, token, and runtime controls wi ["remote access save", "button-save-remote-access"], ["repo sync action", "button-sync-repos"], ["fallback toggle", "checkbox-fallback-to-next-coding-agent"], + ["second-model review toggle", "checkbox-second-model-review"], + ["review model selector", "select-review-model"], ["auto fix conflicts toggle", "checkbox-auto-resolve-conflicts"], ["auto update docs toggle", "checkbox-auto-update-docs"], ["runtime drain button", "button-toggle-drain"], diff --git a/client/src/pages/settings.tsx b/client/src/pages/settings.tsx index 8266d4c..4a48d35 100644 --- a/client/src/pages/settings.tsx +++ b/client/src/pages/settings.tsx @@ -2,7 +2,8 @@ import { useEffect, useState, type ReactNode } from "react"; import { useMutation, useQuery } from "@tanstack/react-query"; import { queryClient, apiRequest } from "@/lib/queryClient"; import { getRepoHref } from "@/lib/repoHref"; -import type { Config, ReleaseRun, RuntimeState, WatchedRepo } from "@shared/schema"; +import { buildReviewModelOptions, findReviewModelSelection } from "@/lib/agentModelOptions"; +import type { AgentModelCatalog, AgentModelOption, Config, ReleaseRun, RuntimeState, WatchedRepo } from "@shared/schema"; import { AppHeader } from "@/components/AppHeader"; import { UpdateBanner } from "@/components/UpdateBanner"; import { toast } from "@/hooks/use-toast"; @@ -50,21 +51,20 @@ const DEFAULT_SETTING_VALUES = { deploymentCheckPollSeconds: 60, }; -const CODEX_MODEL_OPTIONS = [ - { value: "", label: "CLI default" }, - { value: "gpt-5.5", label: "gpt-5.5" }, - { value: "gpt-5.4", label: "gpt-5.4" }, - { value: "gpt-5.4-mini", label: "gpt-5.4-mini" }, - { value: "gpt-5.3-codex", label: "gpt-5.3-codex" }, - { value: "gpt-5.3-codex-spark", label: "gpt-5.3-codex-spark" }, - { value: "gpt-5.2", label: "gpt-5.2" }, -] as const; +const EMPTY_AGENT_MODEL_CATALOG: AgentModelCatalog = { + codex: [{ value: "", label: "CLI default" }], + claude: [{ value: "", label: "CLI default" }], +}; -const CLAUDE_MODEL_OPTIONS = [ - { value: "", label: "CLI default" }, - { value: "opus", label: "opus" }, - { value: "sonnet", label: "sonnet" }, -] as const; +function includeSelectedModel( + options: readonly AgentModelOption[], + selected: string | null | undefined, +): AgentModelOption[] { + if (!selected || options.some((option) => option.value === selected)) { + return [...options]; + } + return [...options, { value: selected, label: selected }]; +} const CODEX_REASONING_OPTIONS = [ { value: "default", label: "CLI default" }, @@ -406,6 +406,23 @@ export default function Settings() { const { data: config } = useQuery({ queryKey: ["/api/config"], }); + const { data: agentModelCatalog } = useQuery({ + queryKey: ["/api/agent-models"], + staleTime: 60_000, + }); + const modelCatalog = agentModelCatalog ?? EMPTY_AGENT_MODEL_CATALOG; + const codexModelOptions = includeSelectedModel(modelCatalog.codex, config?.codexModel); + const claudeModelOptions = includeSelectedModel(modelCatalog.claude, config?.claudeModel); + const primaryModel = config?.codingAgent === "codex" + ? config.codexModel + : config?.claudeModel ?? ""; + const reviewModelOptions = buildReviewModelOptions(modelCatalog, { + agent: config?.codingAgent ?? "claude", + model: primaryModel, + }); + const selectedReviewOption = reviewModelOptions.find((option) => + option.agent === config?.reviewAgent && option.model === config.reviewModel + ); const [newGithubToken, setNewGithubToken] = useState(""); const [showTokenInput, setShowTokenInput] = useState(false); @@ -494,6 +511,7 @@ export default function Settings() { }, onSuccess: () => { queryClient.invalidateQueries({ queryKey: ["/api/config"] }); + queryClient.invalidateQueries({ queryKey: ["/api/agent-models"] }); queryClient.invalidateQueries({ queryKey: ["/api/onboarding/status"] }); queryClient.invalidateQueries({ queryKey: ["/api/github-auth/status"] }); toast({ description: "Settings saved." }); @@ -1044,7 +1062,7 @@ export default function Settings() { className="border border-border bg-transparent px-2 py-1 text-body focus:border-primary focus:outline-none focus-visible:ring-1 focus-visible:ring-ring focus-visible:ring-offset-1 focus-visible:ring-offset-background disabled:opacity-50" > - {CODEX_MODEL_OPTIONS.filter((option) => option.value !== "").map((option) => ( + {includeSelectedModel(codexModelOptions, repo.codexModel).filter((option) => option.value !== "").map((option) => ( ))} @@ -1090,7 +1108,7 @@ export default function Settings() { className="border border-border bg-transparent px-2 py-1 text-body focus:border-primary focus:outline-none focus-visible:ring-1 focus-visible:ring-ring focus-visible:ring-offset-1 focus-visible:ring-offset-background disabled:opacity-50" > - {CLAUDE_MODEL_OPTIONS.filter((option) => option.value !== "").map((option) => ( + {includeSelectedModel(claudeModelOptions, repo.claudeModel).filter((option) => option.value !== "").map((option) => ( ))} @@ -1187,7 +1205,7 @@ export default function Settings() { disabled={updateConfigMutation.isPending} className="border border-border bg-transparent px-2 py-1 text-body focus:border-primary focus:outline-none focus-visible:ring-1 focus-visible:ring-ring focus-visible:ring-offset-1 focus-visible:ring-offset-background disabled:opacity-50" > - {CODEX_MODEL_OPTIONS.map((option) => ( + {codexModelOptions.map((option) => ( ))} @@ -1219,7 +1237,7 @@ export default function Settings() { disabled={updateConfigMutation.isPending} className="border border-border bg-transparent px-2 py-1 text-body focus:border-primary focus:outline-none focus-visible:ring-1 focus-visible:ring-ring focus-visible:ring-offset-1 focus-visible:ring-offset-background disabled:opacity-50" > - {CLAUDE_MODEL_OPTIONS.map((option) => ( + {claudeModelOptions.map((option) => ( ))} @@ -1241,6 +1259,50 @@ export default function Settings() { +
+ +
+ + +
+