Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion packages/cli/src/commands/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ export function runCheck(
allowed: false,
reason:
"HEAD tree changed since pass — amended commits or extra work on tip?",
next: "know-code status",
// Status only diagnoses this condition. A new taught seal starts the
// required state-changing flow for the changed tip.
next: "know-code taught",
};
}

Expand Down
37 changes: 33 additions & 4 deletions packages/cli/src/commands/grade.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
import {
assertGradeProposalForHash,
proposalDigest,
writeGradeProposal,
type GradeProposal,
} from "../grading.js";
import { resolveQuizContext } from "../hash.js";
Expand Down Expand Up @@ -75,7 +76,7 @@ function buildGradeReceipt(
return receipt;
}

export function cmdGradePropose(opts: { json?: boolean }): void {
export function cmdGradePropose(opts: { json?: boolean; write?: boolean }): void {
const repoRoot = findGitRoot();
const config = readConfig(repoRoot);
const ctx = resolveQuizContext(repoRoot, config);
Expand All @@ -87,6 +88,10 @@ export function cmdGradePropose(opts: { json?: boolean }): void {
console.error(err instanceof Error ? err.message : err);
process.exit(1);
}
const answersDigest = answers.answersDigest;
if (!answersDigest) {
throw new Error("know-code: answers.json missing answers digest");
}

let quiz: QuizSpec | null = null;
const qPath = quizPath(repoRoot);
Expand All @@ -96,7 +101,7 @@ export function cmdGradePropose(opts: { json?: boolean }): void {

const context = {
diffHash: ctx.diffHash,
answersDigest: answers.answersDigest,
answersDigest,
level: resolveLevel(repoRoot, answers.level),
scope: ctx.scope,
passScore: PASS_SCORE,
Expand All @@ -107,7 +112,7 @@ export function cmdGradePropose(opts: { json?: boolean }): void {
proposalSchema: {
version: 1,
diffHash: ctx.diffHash,
answersDigest: answers.answersDigest,
answersDigest,
proposedScore: 0.85,
passed: true,
perQuestion: (quiz?.questions ?? []).map((q) => ({
Expand All @@ -121,6 +126,29 @@ export function cmdGradePropose(opts: { json?: boolean }): void {
},
};

if (opts.write) {
writeGradeProposal(repoRoot, {
version: 1,
diffHash: ctx.diffHash,
answersDigest,
proposedScore: 0,
passed: false,
perQuestion: (quiz?.questions ?? []).map((q) => ({
id: q.id,
score: 0,
feedback: "Replace with an evidence-based assessment.",
})),
rubricVersion: "1",
gradedBy: "agent-template",
gradedAt: new Date().toISOString(),
level: resolveLevel(repoRoot, answers.level),
});
console.log(
"know-code: wrote a failing grade-proposal template; the agent must assess every answer before human review.",
);
return;
}

if (opts.json) {
console.log(JSON.stringify(context, null, 2));
return;
Expand Down Expand Up @@ -283,9 +311,10 @@ export async function cmdGrade(opts: {
review?: boolean;
accept?: boolean;
json?: boolean;
write?: boolean;
}): Promise<void> {
if (opts.subcommand === "propose") {
cmdGradePropose({ json: opts.json });
cmdGradePropose({ json: opts.json, write: opts.write });
return;
}

Expand Down
31 changes: 31 additions & 0 deletions packages/cli/src/e2e-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
writeFile,
commitAll,
liteConfig,
setupOpenGate,
writeCommitEditMsg,
} from "./test-helpers.js";
import { messageWithTrailer } from "./trailers.js";
Expand Down Expand Up @@ -65,6 +66,36 @@ describe("e2e workflows", () => {
}
});

it("requireTrailer denial points to commit, which supplies the grounded pending trailer", () => {
const { root, cleanup } = withTempRepo("kc-e2e-trailer-next-");
try {
const { hash } = setupOpenGate(root, { requireTrailer: true });
const denied = runCheck(root);
assert.equal(denied.allowed, false);
assert.equal(denied.next, 'know-code commit -m "…"');

writeCommitEditMsg(root, injectTrailer(["-m", "feat: change"], hash)[1]);
assert.equal(runCheck(root).allowed, true);
} finally {
cleanup();
}
});

it("a pushed HEAD changed after pass points to the state-changing re-teach flow", () => {
const { root, cleanup } = withTempRepo("kc-e2e-head-next-");
try {
setupOpenGate(root, { requireTrailer: false });
writeFile(root, "after-pass.txt", "new tip\n");
commitAll(root, "feat: after pass");

const denied = runCheck(root, { push: true });
assert.equal(denied.allowed, false);
assert.equal(denied.next, "know-code taught");
} finally {
cleanup();
}
});

it("range begin → commits → verify grounded tip trailer", () => {
const { root, cleanup } = withTempRepo("kc-e2e-range-");
try {
Expand Down
11 changes: 8 additions & 3 deletions packages/cli/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import { CANONICAL_FLOW } from "./grading.js";
import { findGitRoot } from "./paths.js";
import { uninstallGitHooks, uninstallAgentHooks, cmdHooksInstall } from "./hooks.js";
import { cmdOverride } from "./override.js";
import { cmdQuestions } from "./questions.js";
import { cmdQuestions, cmdQuizInit } from "./questions.js";
import { cmdAttestInit } from "./seal.js";

function packageVersion(): string {
Expand Down Expand Up @@ -58,7 +58,7 @@ Usage:
know-code doctor [--json] [--strict]
know-code range begin|status|seal|abort|continue [--from <ref>] [--rewrite] [--keep-seal] [--yes]
know-code questions [--json] [--template] [--from <ref>] [--level …]
know-code quiz validate [--path .know-code/quiz.json] [--json]
know-code quiz init|validate [--path .know-code/quiz.json] [--json]
know-code taught [--skip] [--hash <diffHash>] [--passphrase <secret>]
know-code ask [--quiz .know-code/quiz.json] [--port 3847] [--timeout 1800] [--no-open]
know-code grade propose [--json]
Expand Down Expand Up @@ -277,14 +277,18 @@ function main(): void {
});
break;
case "quiz":
if (subcommand === "init") {
cmdQuizInit();
break;
}
if (subcommand === "validate") {
cmdQuizValidate({
path: typeof flags.path === "string" ? flags.path : undefined,
json: flags.json === true,
});
break;
}
console.error("know-code quiz: use validate\n");
console.error("know-code quiz: use init | validate\n");
process.exit(1);
break;
case "check":
Expand Down Expand Up @@ -313,6 +317,7 @@ function main(): void {
review: flags.review === true,
accept: flags.accept === true,
json: flags.json === true,
write: flags.write === true,
}).catch(failAsync);
return;
case "pass":
Expand Down
176 changes: 176 additions & 0 deletions packages/cli/src/pipeline-hints.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { mkdirSync } from "node:fs";
import { join } from "node:path";

import { writeAnswers, writeGrade, writeTaught } from "./attest.js";
import { writeConfig } from "./config.js";
import { readGateSafe } from "./gate.js";
import { computeDiffContext, resolveQuizContext } from "./hash.js";
import { evaluatePipeline, formatCheckDeny } from "./pipeline.js";
import { commitAll, git, liteConfig, setupOpenGate, withTempRepo, writeFile } from "./test-helpers.js";

function blockerCommand(repoRoot: string, step: string): string | undefined {
return evaluatePipeline(repoRoot).blockers.find((blocker) => blocker.step === step)
?.command;
}

function seedWorkflowArtifacts(repoRoot: string): { hash: string; cfg: ReturnType<typeof liteConfig> } {
const cfg = liteConfig({ requireGradeProposal: false });
writeConfig(repoRoot, cfg);
const hash = computeDiffContext(repoRoot, cfg).diffHash;
writeTaught(repoRoot, {
version: 1,
diffHash: hash,
taughtAt: new Date().toISOString(),
skipped: false,
});
writeFile(
repoRoot,
".know-code/quiz.json",
JSON.stringify({
diffHash: hash,
level: "lite",
title: "quiz",
questions: [
{ id: "q1", prompt: "What changed?" },
{ id: "q2", prompt: "Why?" },
],
}),
);
writeAnswers(repoRoot, {
diffHash: hash,
answers: [
{ id: "q1", answer: "A" },
{ id: "q2", answer: "B" },
],
});
writeGrade(repoRoot, {
version: 1,
diffHash: hash,
score: 1,
passed: true,
gradedAt: new Date().toISOString(),
answersDigest: "seeded",
});
return { hash, cfg };
}

describe("pipeline recovery hints", () => {
it("uses a runnable scaffold command when quiz.json is missing", () => {
const { root, cleanup } = withTempRepo("kc-hint-quiz-");
try {
writeFile(root, "a.txt", "base\n");
commitAll(root, "base");
mkdirSync(join(root, ".know-code"), { recursive: true });
writeConfig(root, liteConfig());

assert.equal(blockerCommand(root, "quiz"), "know-code quiz init");
} finally {
cleanup();
}
});

it("uses a runnable proposal-draft command when a grade proposal is missing", () => {
const { root, cleanup } = withTempRepo("kc-hint-proposal-");
try {
writeFile(root, "a.txt", "base\n");
commitAll(root, "base");
mkdirSync(join(root, ".know-code"), { recursive: true });
writeConfig(root, liteConfig({ requireGradeProposal: true }));

assert.equal(
blockerCommand(root, "grade-proposal"),
"know-code grade propose --write",
);
} finally {
cleanup();
}
});

it("maps every pipeline recovery state to a state-changing command", () => {
const { root, cleanup } = withTempRepo("kc-hint-matrix-");
try {
writeFile(root, "a.txt", "base\n");
commitAll(root, "base");
mkdirSync(join(root, ".know-code"), { recursive: true });
seedWorkflowArtifacts(root);

const commands = evaluatePipeline(root).blockers
.map((blocker) => blocker.command)
.filter((command): command is string => !!command);
for (const command of commands) {
assert.doesNotMatch(command, /know-code status|&& write|^Agent:/);
}
assert.ok(commands.every((command) => command.startsWith("know-code ") || command === "git add -u"));
} finally {
cleanup();
}
});

it("covers every emitted pipeline recovery command", () => {
const seen = new Set<string>();
const check = (setup: (root: string) => void, step: string, command: string) => {
const { root, cleanup } = withTempRepo("kc-hint-command-");
try {
writeFile(root, "a.txt", "base\n");
commitAll(root, "base");
mkdirSync(join(root, ".know-code"), { recursive: true });
setup(root);
assert.equal(blockerCommand(root, step), command);
seen.add(command);
} finally {
cleanup();
}
};

check((root) => writeConfig(root, liteConfig({ requireAttest: true })), "attest", "know-code attest-init");
check((root) => writeConfig(root, liteConfig({ rangeMode: "range" })), "range", "know-code range begin");
check((root) => writeConfig(root, liteConfig()), "taught", "know-code taught");
check((root) => writeConfig(root, liteConfig()), "quiz", "know-code quiz init");
check((root) => writeConfig(root, liteConfig()), "answers", "know-code ask");
check((root) => writeConfig(root, liteConfig({ requireGradeProposal: true })), "grade-proposal", "know-code grade propose --write");
check((root) => writeConfig(root, liteConfig()), "grade", "know-code grade --review");
check((root) => writeConfig(root, liteConfig()), "pass", "know-code pass");

check((root) => {
writeConfig(root, liteConfig());
writeFile(root, ".know-code/taught.json", "{");
}, "corrupt", "know-code reset");

check((root) => {
setupOpenGate(root, { requireTrailer: false });
writeFile(root, "a.txt", "dirty\n");
}, "pass", "git add -u");

assert.deepEqual([...seen].sort(), [
"git add -u",
"know-code ask",
"know-code attest-init",
"know-code grade --review",
"know-code grade propose --write",
"know-code pass",
"know-code quiz init",
"know-code range begin",
"know-code reset",
"know-code taught",
]);
});

it("keeps a stale-pass deny reason aligned with its pass recovery command", () => {
const { root, cleanup } = withTempRepo("kc-hint-stale-pass-");
try {
const { cfg } = setupOpenGate(root, { requireTrailer: false });
writeFile(root, "a.txt", "changed\n");
git(root, ["add", "a.txt"]);
seedWorkflowArtifacts(root);

const ctx = resolveQuizContext(root, cfg);
const denied = formatCheckDeny(root, cfg, ctx, readGateSafe(root));
assert.equal(denied.next, "know-code pass");
assert.match(denied.reason, /Run `know-code pass`/);
} finally {
cleanup();
}
});
});
Loading