Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ jobs:
with:
name: web-live-route-smoke-evidence
path: |
docs/workhub/05-clients/assets/audit/2026-06-11-r4-web-live-route-interaction/contact-sheet.png
docs/workhub/05-clients/assets/audit/2026-06-11-r4-web-live-route-interaction/live-route-interaction-report.json
docs/workhub/05-clients/assets/audit/2026-06-11-r4-web-live-route-interaction/smoke-summary.md
artifacts/qa/r4-web-live-route-interaction/contact-sheet.png
artifacts/qa/r4-web-live-route-interaction/live-route-interaction-report.json
artifacts/qa/r4-web-live-route-interaction/smoke-summary.md
retention-days: 14

rust-system-i18n:
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# 参考代码:仅本地保留,绝不入库(当前项目源码 + opencode)
/reference/

# 生成型 QA 审计产物(报告/截图):可复跑生成,绝不入库——verify 末尾以 git diff --exit-code 保证干净
/artifacts/

# 依赖与构建产物
node_modules/
dist/
Expand Down
19 changes: 8 additions & 11 deletions apps/api/src/qa/r14-chat-smoke.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
// R14 批 CHAT 真库冒烟(一次性可复跑验证脚本,照 r12-real-key-smoke.ts 的定位与防呆惯例):
// 在专用 scratch 库上跑完整迁移链后,走服务层把聊天完整度的全链路过一遍——
// 在自建唯一命名的 scratch 库上跑完整迁移链后,走服务层把聊天完整度的全链路过一遍——
// 引用回复→编辑(含他人编辑 403)→reaction 幂等加减→置顶/取消→已读游标单调夹紧+receipts→
// 墓碑删除(含引用侧墓碑联动)→SSE 事件(recording bus 断言三个新事件都真的发布了)。
// 不需要 LLM key。需要环境:DATABASE_URL(专用 scratch 库!命名必须匹配 workhub_r14_*smoke)。
// 不需要 LLM key。需要环境:DATABASE_URL(workhub_r14_*smoke 命名的锚库,只用来建/删唯一库);
// smoke 每次 CREATE 唯一库、结束 DROP——与 search 等其他真库 smoke 互不污染。
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";

import { loadSettings } from "@workhub/config";
import {
createConversationRepository,
createDatabaseClient,
Expand All @@ -21,16 +21,14 @@ import {
} from "@workhub/db";

import { createConversationService } from "../services/conversations.js";
import { withR14SmokeDatabase, type R14SmokeSettings } from "./r14-smoke-db.js";

async function main() {
const settings = loadSettings(process.env);
if (settings.appEnv === "production") {
throw new Error("Refusing to run the R14 chat smoke in production.");
}
if (!/workhub_r14_[a-z0-9_]*smoke/u.test(settings.databaseUrl)) {
throw new Error("R14 chat smoke requires a dedicated workhub_r14_*smoke scratch database.");
}
await withR14SmokeDatabase("chat", runChatSmoke);
process.exit(0);
}

async function runChatSmoke(settings: R14SmokeSettings) {
await runMigrations(settings);
const client = createDatabaseClient(settings);
const db = client.db;
Expand Down Expand Up @@ -222,7 +220,6 @@ async function main() {
})
);
await client.close?.();
process.exit(0);
}

main().catch((error) => {
Expand Down
23 changes: 10 additions & 13 deletions apps/api/src/qa/r14-search-smoke.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
// R14 批 SEARCH 真库冒烟(一次性可复跑验证脚本,照 r14-chat-smoke.ts / r12-real-key-smoke.ts 的定位与防呆惯例):
// 在专用 scratch 库上跑完整迁移链(含 0057 pg_trgm + 5 GIN 索引)后,直接种四数据源 + 一个「他人的个人空间
// 项目」作围栏反例,再用 search 服务把四 scope 全过一遍——断言命中、墓碑滤除、个人空间围栏、assignee EXISTS、
// CJK 2 字/≥3 字、has_more、LIKE 元字符转义、空结果诚实。不需要 LLM key。
// 需要环境:DATABASE_URL(专用 scratch 库!命名必须匹配 workhub_r14_*smoke)。
// 在自建唯一命名的 scratch 库上跑完整迁移链(含 0057 pg_trgm + 5 GIN 索引)后,直接种四数据源 + 一个「他人的
// 个人空间项目」作围栏反例,再用 search 服务把四 scope 全过一遍——断言命中、墓碑滤除、个人空间围栏、assignee
// EXISTS、CJK 2 字/≥3 字、has_more、LIKE 元字符转义、空结果诚实。不需要 LLM key。
// 需要环境:DATABASE_URL(workhub_r14_*smoke 命名的锚库,只用来建/删唯一库);smoke 每次 CREATE 唯一库、
// 结束 DROP——chat smoke 先写过「完播率」消息也污染不到这里的固定命中数断言。
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";

import { loadSettings } from "@workhub/config";
import {
conversationMessages,
createDatabaseClient,
Expand All @@ -28,16 +28,14 @@ import {

import type { AuthActor } from "../middleware/auth.js";
import { createSearchService } from "../services/search.js";
import { withR14SmokeDatabase, type R14SmokeSettings } from "./r14-smoke-db.js";

async function main() {
const settings = loadSettings(process.env);
if (settings.appEnv === "production") {
throw new Error("Refusing to run the R14 search smoke in production.");
}
if (!/workhub_r14_[a-z0-9_]*smoke/u.test(settings.databaseUrl)) {
throw new Error("R14 search smoke requires a dedicated workhub_r14_*smoke scratch database.");
}
await withR14SmokeDatabase("search", runSearchSmoke);
process.exit(0);
}

async function runSearchSmoke(settings: R14SmokeSettings) {
await runMigrations(settings);
const client = createDatabaseClient(settings);
const db = client.db;
Expand Down Expand Up @@ -239,7 +237,6 @@ async function main() {
})
);
await client.close?.();
process.exit(0);
}

main().catch((error) => {
Expand Down
58 changes: 58 additions & 0 deletions apps/api/src/qa/r14-smoke-db.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// R14 真库冒烟的隔离库辅助:以操作者提供的 workhub_r14_*smoke 库为锚(只用来 CREATE/DROP,不写业务数据),
// 每次运行自建唯一命名的 scratch 库,跑完(无论成败)DROP 清理——chat/search 等多个 smoke 互不污染,
// 「固定命中数」类断言不再受运行顺序影响。
// 守卫不降级:production 拒跑 + 锚库命名必须匹配 workhub_r14_*smoke,两条都保留在入口。
import { randomUUID } from "node:crypto";

import { loadSettings } from "@workhub/config";
import { createDatabaseClient } from "@workhub/db";

export type R14SmokeSettings = ReturnType<typeof loadSettings>;

function swapDatabaseName(databaseUrl: string, databaseName: string) {
const url = new URL(databaseUrl);
url.pathname = `/${databaseName}`;
return url.toString();
}

async function createUniqueDatabase(admin: ReturnType<typeof createDatabaseClient>, name: string) {
// 并发 smoke 同时从 template1 建库会撞「source database is being accessed by other users」——短退避重试。
let lastError: unknown;
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
await admin.pool.query(`CREATE DATABASE "${name}"`);
return;
} catch (error) {
lastError = error;
await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1)));
}
}
throw lastError;
}

export async function withR14SmokeDatabase<T>(
smokeName: string,
run: (settings: R14SmokeSettings) => Promise<T>
): Promise<T> {
const anchorSettings = loadSettings(process.env);
if (anchorSettings.appEnv === "production") {
throw new Error(`Refusing to run the R14 ${smokeName} smoke in production.`);
}
if (!/workhub_r14_[a-z0-9_]*smoke/u.test(anchorSettings.databaseUrl)) {
throw new Error(`R14 ${smokeName} smoke requires a dedicated workhub_r14_*smoke scratch anchor database.`);
}
const uniqueName = `workhub_r14_${smokeName}_smoke_${randomUUID().replace(/-/gu, "").slice(0, 12)}`;
const admin = createDatabaseClient(anchorSettings);
try {
await createUniqueDatabase(admin, uniqueName);
try {
const derivedSettings = loadSettings({ ...process.env, DATABASE_URL: swapDatabaseName(anchorSettings.databaseUrl, uniqueName) });
return await run(derivedSettings);
} finally {
// WITH (FORCE)(PG13+):断言失败时残留连接也不挡清理。
await admin.pool.query(`DROP DATABASE IF EXISTS "${uniqueName}" WITH (FORCE)`);
}
} finally {
await admin.close();
}
}
9 changes: 7 additions & 2 deletions apps/api/src/qa/r5-10-real-key-evaluation-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,13 @@ export function createR5_10WorkItemServiceOptions(
};
}

export function createR5_10ClarificationAnswerPayload(clarificationAnswer: string) {
return { free_text: clarificationAnswer };
export function createR5_10ClarificationAnswerPayload(clarificationAnswer: string, selectedOptionId?: string) {
// R10 option-first 契约:有候选时选中一个真实候选——id 只能来自服务端问题卡,绝不是任务预设;
// free_text 始终携带确认语(诚实退化成 long_text 时它就是全部答案)。
return {
...(selectedOptionId ? { selected_option_ids: [selectedOptionId] } : {}),
free_text: clarificationAnswer
};
}

export function selectR5_10TasksForRun<T>(allTasks: readonly T[], rawTaskLimit: string | undefined) {
Expand Down
35 changes: 29 additions & 6 deletions apps/api/src/qa/r5-10-real-key-evaluation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ type ClarificationQuestionEvidence = {
body?: string;
input_mode: string;
options?: { id?: string; label?: string }[];
recommended_option_ids?: string[];
};

type EvalClarificationFileContext = {
Expand Down Expand Up @@ -193,6 +194,7 @@ type EvalTaskReport = {
title: string;
body: string | null;
option_count: number;
selected_option_id: string | null;
answer: string;
};
artifacts: {
Expand Down Expand Up @@ -317,11 +319,25 @@ function clarificationAnswerFor(task: EvalTask) {

function assertRealClarificationQuestion(task: EvalTask, question: ClarificationQuestionEvidence) {
const options = question.options ?? [];
if (question.input_mode !== "long_text") {
throw new Error(`${task.id} expected AI clarification to request free text, got ${question.input_mode}.`);
}
if (options.length > 0) {
throw new Error(`${task.id} expected AI clarification to avoid preset choices, got ${options.length} options.`);
// R10-0c(P1-1)option-first 契约:LLM 草稿带 ≥2 个候选答案时渲 single_choice 选项卡(自由文本折叠兜底);
// 没有可用候选时诚实退化为 long_text(不造假选项)。两种形态都必须是「真问题」,反模板门在下方保留。
if (question.input_mode === "single_choice") {
if (options.length < 2) {
throw new Error(`${task.id} option-first clarification must offer at least 2 candidates, got ${options.length}.`);
}
const ids = options.map((option) => option.id ?? "");
if (ids.some((id) => !id.trim()) || new Set(ids).size !== ids.length) {
throw new Error(`${task.id} option-first clarification has empty or duplicated option ids.`);
}
if (options.some((option) => !(option.label ?? "").trim())) {
throw new Error(`${task.id} option-first clarification has an empty option label.`);
}
} else if (question.input_mode === "long_text") {
if (options.length > 0) {
throw new Error(`${task.id} long_text degradation must not fabricate preset choices, got ${options.length} options.`);
}
} else {
throw new Error(`${task.id} expected single_choice (option-first) or long_text clarification, got ${question.input_mode}.`);
}
const combined = `${question.title}\n${question.body ?? ""}`.toLowerCase();
const genericPresetCount = [
Expand Down Expand Up @@ -889,10 +905,16 @@ async function main() {
const clarificationQuestion = session.data.question;
assertRealClarificationQuestion(task, clarificationQuestion);
const clarificationAnswer = clarificationAnswerFor(task);
// option-first 时选中真实候选(优先推荐项,否则第一项);long_text 退化时只带 free_text。
const clarificationOptions = clarificationQuestion.options ?? [];
const recommendedOptionId = clarificationQuestion.recommended_option_ids?.[0];
const selectedClarificationOptionId = clarificationQuestion.input_mode === "single_choice"
? clarificationOptions.find((option) => option.id === recommendedOptionId)?.id ?? clarificationOptions[0]?.id
: undefined;
await requestJson(
"POST",
`/api/sessions/${session.data.session_id}/next-question`,
createR5_10ClarificationAnswerPayload(clarificationAnswer),
createR5_10ClarificationAnswerPayload(clarificationAnswer, selectedClarificationOptionId),
200
);
const createdWorkItem = await requestJson<{ data: { workitem: { id: string; status: string } } }>("POST", "/api/workitems", {
Expand Down Expand Up @@ -1086,6 +1108,7 @@ async function main() {
title: clarificationQuestion.title,
body: clarificationQuestion.body ?? null,
option_count: clarificationQuestion.options?.length ?? 0,
selected_option_id: selectedClarificationOptionId ?? null,
answer: clarificationAnswer
},
artifacts: {
Expand Down
16 changes: 9 additions & 7 deletions apps/api/src/work-items-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -942,13 +942,15 @@ test("real-key evaluation wires the provider registry into WorkItem clarificatio
assert.deepEqual(await options.projectFileContext?.({ intentText: "没有预置文件的任务" }), []);
});

test("real-key evaluation answers AI clarification with free text before applying task presets", () => {
const payload = createR5_10ClarificationAnswerPayload("请优先输出适合项目验收的要点。");

// R9.7: the old assertion grepped request-body source around `/next-question`.
// That was wrong because source text did not prove the clarification payload omits preset option ids.
assert.deepEqual(payload, { free_text: "请优先输出适合项目验收的要点。" });
assert.equal(Object.hasOwn(payload, "selected_option_ids"), false);
test("real-key evaluation answers AI clarification per the R10 option-first contract", () => {
// long_text 诚实退化:只带 free_text,绝不携带任务预设 option id。
const freeTextOnly = createR5_10ClarificationAnswerPayload("请优先输出适合项目验收的要点。");
assert.deepEqual(freeTextOnly, { free_text: "请优先输出适合项目验收的要点。" });
assert.equal(Object.hasOwn(freeTextOnly, "selected_option_ids"), false);

// single_choice option-first:选中服务端问题卡给出的真实候选 id,free_text 仍携带确认语。
const optionFirst = createR5_10ClarificationAnswerPayload("确认按该口径执行。", "option-2");
assert.deepEqual(optionFirst, { selected_option_ids: ["option-2"], free_text: "确认按该口径执行。" });
});

test("real-key evaluation labels limited samples instead of applying full-suite gates", () => {
Expand Down
11 changes: 2 additions & 9 deletions apps/web/qa/r4-web-live-route-interaction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -364,15 +364,8 @@ type RouteStatusProbe = {
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(scriptDir, "../../..");
const webRoot = path.join(repoRoot, "apps", "web");
const defaultOutputDir = path.join(
repoRoot,
"docs",
"workhub",
"05-clients",
"assets",
"audit",
"2026-06-11-r4-web-live-route-interaction"
);
// 生成型审计产物落 gitignored artifacts 区:失败中断时删掉的只是生成物,tracked 历史证据不受影响。
const defaultOutputDir = path.join(repoRoot, "artifacts", "qa", "r4-web-live-route-interaction");
const r4ReactComponentByRoute: Record<string, string> = {
home: "HomeRouteComponent",
proposal: "ProposalRouteComponent",
Expand Down
49 changes: 49 additions & 0 deletions apps/web/src/chrome-launch.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import assert from "node:assert/strict";
import test from "node:test";
import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";

import { launchChrome } from "./chrome-launch.js";

// BUG-09:Chrome 起不来时诊断不能被吞——stderr/退出状态/启动参数/版本/端口探测都要出现在错误报告里,
// 且子进程已退出时立即失败,不傻等整个 CDP 超时窗把真因冲淡成 fetch timeout。
test("launchChrome fails fast with full diagnostics when Chrome exits before the debug port comes up", async () => {
const tmp = await mkdtemp(path.join(os.tmpdir(), "workhub-chrome-launch-diag-"));
try {
const fakeChromePath = path.join(tmp, "fake-chrome.sh");
await writeFile(
fakeChromePath,
[
"#!/bin/sh",
"if [ \"$1\" = \"--version\" ]; then",
" echo \"FakeChrome 0.0.1\"",
" exit 0",
"fi",
"echo \"fake chrome cannot start: boom\" >&2",
"exit 3",
""
].join("\n"),
"utf8"
);
await chmod(fakeChromePath, 0o755);

const startedAt = Date.now();
await assert.rejects(
launchChrome(fakeChromePath, 65533, path.join(tmp, "profile"), { debugTargetTimeoutMs: 20_000 }),
(error: Error) => {
assert.match(error.message, /Chrome exited \(code 3, signal null\) before the CDP debug port 65533 came up/u);
assert.match(error.message, /Chrome launch diagnostics:/u);
assert.match(error.message, /fake chrome cannot start: boom/u, "stderr tail is preserved");
assert.match(error.message, /FakeChrome 0\.0\.1/u, "chrome --version is captured");
assert.match(error.message, /--remote-debugging-port=65533/u, "launch args are reported");
assert.match(error.message, /port_probes/u, "port probe trace is reported");
return true;
}
);
// 子进程秒退,不许烧完 20s 超时窗:快速失败是这条修复的一半。
assert.ok(Date.now() - startedAt < 10_000, "fails fast instead of waiting out the CDP timeout");
} finally {
await rm(tmp, { recursive: true, force: true });
}
});
Loading
Loading