From c7e3000864f8bbd5af21acbd25910f94b9b538ea Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sun, 30 Aug 2026 02:51:21 -0400 Subject: [PATCH 1/3] feat: add KB lifecycle extension contract --- README.md | 69 +- docs/agent-workflow.md | 36 +- docs/design.md | 55 +- scripts/kb-skill-contract.test.ts | 556 +++++++++ scripts/kb-skill-contract.ts | 486 ++++++++ scripts/package-smoke.ts | 3 + skills/kb/AGENTS.md | 3 + skills/kb/SKILL.md | 66 +- skills/kb/agents/openai.yaml | 4 +- skills/kb/references/companion-skills.md | 96 ++ skills/kb/references/customize.md | 123 ++ skills/kb/references/percolate.md | 46 +- .../kb/templates/companion-skill.template.md | 57 + src/authoring.test.ts | 8 +- src/authoring.ts | 8 +- src/cli.test.ts | 29 + src/cli.ts | 19 +- src/graph.test.ts | 10 + src/graph.ts | 9 +- src/percolate.test.ts | 142 ++- src/percolate.ts | 1027 ++++++++++++++++- 21 files changed, 2777 insertions(+), 75 deletions(-) create mode 100644 scripts/kb-skill-contract.test.ts create mode 100644 scripts/kb-skill-contract.ts create mode 100644 skills/kb/references/companion-skills.md create mode 100644 skills/kb/references/customize.md create mode 100644 skills/kb/templates/companion-skill.template.md diff --git a/README.md b/README.md index fa7bda3..958a484 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,14 @@ Retrieval is bounded. The high-level `kb search` and `KnowledgeBaseSession.searc Each note owns its outbound typed relationships in frontmatter. KB derives backlinks, inverse edges, and bounded traversal at read time, so parallel agents do not contend on one generated fact file. `kb percolate ` reports recurring concepts and missing-link candidates with inspectable support but writes nothing. An agent reads the cited notes before creating a reusable concept or relationship. Semantic similarity never creates an edge automatically. +Percolation Result V2 presents a missing relationship as an unordered pair of +notes with a required predicate. It does not choose the source, direction, or a +`related-to` fallback. Recommended authored predicates include `synthesizes`, +`evidenced-by`, `informed-by`, `supersedes`, and `contradicts`; they are an +advisory vocabulary, so a vault can use another canonical predicate when its +prose and evidence define the claim. KB never infers reciprocal, inverse, +transitive, or similarity-derived relationships. + Git provenance is opt-in. A search without `--history` performs no Git indexing. `--history` requests best-effort provenance, while `--require-history` rejects unavailable history or incomplete provenance for the selected notes. If one commit exceeds the 2,000-path detail limit, KB retains its identity and vault-local note associations, marks its co-change detail incomplete, and continues through later commits. Best-effort search reports that requested lane as partial. Local attachment checks cover Markdown and Obsidian references to images, PDFs, and editable tldraw sources. They reject missing or escaping files while leaving external URLs alone. A source-inbox view separately lists recent captures that have no inbound disposition from maintained knowledge. It is an advisory, not an automatic backlink requirement: a saved source may intentionally remain a leaf. @@ -187,6 +195,39 @@ The same mixed-cache, single-run test recorded p95 latencies of 44.345 milliseco Search finds candidates. Similarity does not establish that a passage is current, correct, or supported by its sources. The Markdown, cited captures, explicit relationships, and requested Git history supply the material a reader must inspect. +### Customize through an approved proposal + +The Agent Skill routes setup and evolution requests before it prepares a +runtime. It inspects the proposed location without mutation, interviews the +user about the memory questions the KB should answer, and presents exact read +and write targets. Only the approved targets may be scaffolded. A changed path, +repository, account, integration, or companion skill requires renewed +approval. + +The standard router may be enough. A recurring ritual can instead receive a +companion skill with explicit inputs, authority, durable outputs, idempotence, +failure behavior, and verification. These skills are inert instructions. They +do not create a plugin runtime, execute vault metadata, inherit ambient account +access, or couple application code to the KB. An exact repeat is a no-op; +divergence, path escape, symbolic links, partial writes, and unapproved +external surfaces stop the workflow. + +The repository's fake-capability suite exercises those transitions. It is a +tested contract example, not proof that every agent or host integration +complies. + +This workflow builds on Frank Chen's public notes about [designing a personal +knowledge base with an +agent](https://gist.github.com/fxchen/773397095d7a6bffda621e4237da0da9) +and [extending it with +skills](https://gist.github.com/fxchen/09cb410b22c9c5256d80243ee925b57e). + +KB ships no `kb_role` metadata, lifecycle resolver or API, lifecycle CLI, +compatibility diagnostic, or metadata migration. A frozen Phase 0 value gate +must show that those surfaces improve deterministic agent decisions before they +are introduced. Current and historical plan routing remains derived from +existing type, path, and status conventions. + ### Adopt the smallest useful split Start with a short inherited `AGENTS.md` path for rules whose omission would make an edit wrong. A small knowledge base may need only Markdown, Git, an index page, and ordinary file search. Add source capture when evidence keeps disappearing. Add repository scopes when agents need to recover current memory from code paths. Add metadata or hybrid search when file search stops answering the repository's questions. Add links and graph views only when the relationships themselves help people make decisions. @@ -205,6 +246,11 @@ bytes with status `prepared`, and has no vault, Git, Oh-store, or promotion capability. KB pins `@hraness/oh` v0.2.0 and delegates closure integrity to its official store verifier. +The release also adds interview-first setup guidance, a companion-skill +contract, and Percolation Result V2. V2 requires an explicit predicate instead +of inventing `related-to`, while the V1 parser remains available for historical +results throughout the 0.18 release line. + ## Upgrade to v0.17.3 Version 0.17.3 restructures the README around an inspectable first task, @@ -507,7 +553,12 @@ Predicates use lower-kebab-case. Local targets use exact vault-root IDs without `.md`; cross-vault targets use canonical stable `kb://` URIs. `kb graph`, `kb backlinks`, `kb relation list`, and `kb links` derive inverse edges and bounded paths without injecting reciprocal or inferred facts into notes. `kb percolate` proposes reusable concepts and missing connections with explicit -support; an agent reviews the cited prose before authoring anything. +support; an agent reviews the cited prose before authoring anything. In its V2 +result, a missing relationship is an unordered endpoint pair with a required +predicate, never an executable directed assertion or an automatic +`related-to`. Common reviewed claims use `synthesizes`, `evidenced-by`, +`informed-by`, `supersedes`, or `contradicts`; other canonical custom predicates +remain valid when their meaning is supported. Within a portfolio, a note can target a stable cross-vault identity such as `kb://hraness/sleepyland/sound-wellness-expansion`. The target vault must be @@ -580,8 +631,10 @@ diffs, and the explicit local job ledger are available from The repository ships one reusable `kb` Agent Skill under `skills/kb/`. Its intent router loads focused references only when a task needs them: querying repository context and agent memory, capturing URLs or PDFs, writing durable -plans, promoting concepts and typed relationships, or refreshing and checking -a vault. The package smoke test keeps future tagged packages byte-identical to +plans, promoting concepts and typed relationships, refreshing and checking a +vault, or designing a setup through an interview and approved proposal. An +approved setup may scaffold a bounded companion skill for a distinct recurring +ritual. The package smoke test keeps future tagged packages byte-identical to that source tree. ```sh @@ -591,9 +644,11 @@ bunx skills add hraness/kb#v0.18.0 ``` The skill invokes the installed `kb` command without depending on a repository -checkout. Its runtime setup installs the pinned CLI only when the command is -missing, and it never initializes or mutates a vault as an installation side -effect. The repository's phase-orchestration skill remains available to local -repository agents but is marked internal, so public skill discovery omits it. +checkout. It routes setup and evolution before runtime preparation. For +execution workflows, runtime setup installs the pinned CLI only when the +command is missing, and it never initializes or mutates a vault as an +installation side effect. The repository's phase-orchestration skill remains +available to local repository agents but is marked internal, so public skill +discovery omits it. See [Design](docs/design.md), [Portfolio federation](docs/portfolio.md), [Agent workflow](docs/agent-workflow.md), [PDF capture](docs/pdf.md), and [Contributing](CONTRIBUTING.md) for the durable contracts and development gate. hraness/kb is available under the [MIT License](LICENSE). diff --git a/docs/agent-workflow.md b/docs/agent-workflow.md index 0f90486..9018f62 100644 --- a/docs/agent-workflow.md +++ b/docs/agent-workflow.md @@ -5,6 +5,27 @@ maintaining a vault. Markdown is the durable record. Tool output, catalogs, backlinks, traversed paths, semantic indexes, and percolation candidates are views over that record. +## Customize before preparing a runtime + +When the request is to set up or evolve a KB, design the boundary before +discovering or installing the CLI. Inspect the explicitly proposed repository +and vault location without mutation. A new location does not need an existing +`index.md`. Interview the user about the recurring questions the KB should +answer, then present the exact read and write targets in a proposal. + +The approved proposal may choose the standard Markdown layout, no change, or +zero to three companion skills for recurring rituals. A companion skill must +name its inputs, surfaces, authority, approval boundary, durable outputs, +idempotence, failure behavior, and verification. Skill discovery and ambient +application or account sessions grant no authority. A changed target, +repository, skill, account, or integration requires renewed approval. + +After approval, scaffold only the allowlisted paths. Matching existing bytes +are a no-op. Stop on divergent content, path escape, a symbolic link, partial +failure, or an unapproved external surface. Prepare the runtime only when the +approved operation needs it. Installation never authorizes initialization, +indexing, QMD state, account access, or vault mutation. + ## Orient before editing 1. For a repository-path question, run `kb context`, read the returned @@ -306,9 +327,12 @@ kb relation add notes/write-path supports notes/durable-agent-memory ``` Predicates use lower-kebab-case and targets use exact vault-root IDs without -`.md`. Explain the assertion in prose or evidence. Do not author reciprocal -edges, inferred transitive paths, or relationships derived only from an -embedding score. +`.md`. Recommended predicates for common KB claims are `synthesizes`, +`evidenced-by`, `informed-by`, `supersedes`, and `contradicts`. The list is +advisory; a vault may use another canonical predicate whose meaning its prose +establishes. Explain the assertion in prose or evidence. Do not author inverse, +reciprocal, transitive, or similarity-derived edges. External or unclassified +material does not gain a relationship automatically. ## Capture a source @@ -376,6 +400,12 @@ the write and open a new session after the final check. Open the evidence cited by each percolation candidate. Promote only concepts likely to be reused and relationships established by the source material. +In Percolation Result V2, a missing relationship is an unordered endpoint pair +with a required predicate. It does not choose a source, direction, or +`related-to` fallback. Read both endpoints and author the directed assertion +only when their prose and evidence establish it. Parse historical unversioned +V1 results explicitly; do not silently transform their suggested predicate +into a V2 claim. Review broken and ambiguous links, typed relationships, local attachments, and repository-scope advisories first. Then inspect orphans and high-confidence title or alias mentions in context. Add a suggested link only when it improves diff --git a/docs/design.md b/docs/design.md index 2ee8769..724e54c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -21,6 +21,40 @@ The vault is an ordinary directory of Obsidian-compatible Markdown, suitable for The boundaries separate what a source said from what the vault currently concludes. They are conventions expressed in Markdown and agent guides, not proprietary file formats. +## Setup is an approved instruction workflow + +The packaged Agent Skill routes setup and evolution requests before it prepares +a runtime. It can inspect an explicitly proposed location, interview the user, +and present exact read and write surfaces without installing KB, creating a +vault, building a QMD index, or accessing an ambient account. Only the approved +proposal may scaffold files. A changed path, skill, repository, account, or +integration requires renewed approval. + +A vault may add zero to three companion skills for recurring rituals whose +inputs, authority, durable output, and failure behavior need a distinct +contract. These are inert instruction files. They do not form a runtime plugin +registry, execute vault metadata, inherit account authority, or couple the +vault to application code. An exact repeated scaffold is a no-op; divergent +content, path escape, symbolic links, partial failure, and unapproved external +surfaces stop the workflow. + +The repository models these transitions with fake capabilities and verifies +preapproval zero mutation, exact approved writes, no-op repeats, renewed +approval, divergence, confinement, symbolic links, partial failure, and +external-surface rejection. This is a tested contract example, not proof that +every agent or host integration complies. + +This interview-first setup and bounded extension model builds on Frank Chen's +public notes about [designing a personal knowledge base with an +agent](https://gist.github.com/fxchen/773397095d7a6bffda621e4237da0da9) +and [extending it with skills](https://gist.github.com/fxchen/09cb410b22c9c5256d80243ee925b57e). + +KB ships no `kb_role` field, lifecycle resolver or API, lifecycle CLI, +compatibility diagnostic, or metadata migration. A frozen value gate must show +that those surfaces improve deterministic agent decisions before they are +introduced. Current and historical plan routing remains derived from existing +note type, path, and status. + ## Repository instructions and context have different authority An `AGENTS.md` file is normative, path-scoped, and always loaded before an @@ -161,6 +195,12 @@ without `.md`. The source note is the implicit subject. Different agents can therefore edit relationships on different notes without contending on a central ontology or edge file. +The recommended vocabulary covers common KB claims: `synthesizes`, +`evidenced-by`, `informed-by`, `supersedes`, and `contradicts`. It is advisory, +not a closed ontology. A vault may author another canonical predicate when its +prose and evidence define the claim. Note type, directory, chronology, shared +tags, and semantic similarity do not choose a predicate. + Four rules keep the result honest: 1. Backlinks and inverse relationships are derived, never written into source @@ -170,8 +210,10 @@ Four rules keep the result honest: 3. A title, alias, recurring tag, shared neighborhood, or semantic match is a candidate. It becomes an edge only after an agent or person reviews the evidence and authors the assertion. -4. Transitive paths and other inferred relationships remain query results. - They never silently become Markdown facts. +4. Reciprocal, inverse, transitive, and similarity-derived relationships + remain query results. They never silently become Markdown facts. External or + unclassified material remains unresolved until evidence supports an authored + assertion. This makes inbound and outbound counts, backlinks, relationships, and orphans reproducible. It also prevents reciprocal sections and generated catalogs from @@ -197,6 +239,15 @@ unlinked mentions, and relationship-hygiene findings. The output cites the authored evidence that caused each candidate. A person or agent decides whether to run `kb note create` or `kb relation add`. +Percolation Result V2 emits missing relationships as unordered endpoint pairs +with a required predicate. It does not present either endpoint as the source, +draw a directional edge, or suggest `related-to`. The reviewer reads both +notes, then authors a directed assertion only when the evidence determines its +owner, target, and predicate. Explicit V1 parsers preserve historical +unversioned results for one deprecation cycle; the default parser accepts V2 +only, and no compatibility path guesses a semantic upgrade. V1 remains +available throughout 0.18.x and is not removed before 0.19.0. + This named-command surface is deliberate. Common graph questions receive a small typed contract, deterministic ordering, and an operation-specific bound instead of requiring every agent to construct an ad hoc query program. A diff --git a/scripts/kb-skill-contract.test.ts b/scripts/kb-skill-contract.test.ts new file mode 100644 index 0000000..1897581 --- /dev/null +++ b/scripts/kb-skill-contract.test.ts @@ -0,0 +1,556 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { readFile, readdir } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + customizationProposalDigest, + executeCustomizationContract, + inspectCustomizationContract, + type CustomizationCapabilities, + type CustomizationProposal, + type CustomizationTargetState, + validateKbSkillContractResources, +} from "./kb-skill-contract.ts"; + +const root = "/approved/skills"; + +function proposal( + overrides: Partial = {}, +): CustomizationProposal { + return { + id: "save-decision-kb", + root, + runtime: "none", + reads: [{ surface: "filesystem", target: "/repo/kb" }], + writes: [{ + surface: "filesystem", + target: "save-decision-kb/SKILL.md", + contents: "# Save a decision\n", + }], + ...overrides, + }; +} + +type MutationAttempt = + | { readonly kind: "prepare-runtime" } + | { readonly kind: "write"; readonly target: string }; + +class FakeCapabilities implements CustomizationCapabilities { + readonly inspections: { readonly surface: string; readonly target: string }[] = []; + readonly mutations: MutationAttempt[] = []; + readonly files = new Map(); + readonly states = new Map(); + failRuntimePreparation = false; + failWriteTarget: string | null = null; + + async inspect(surface: string, target: string): Promise { + this.inspections.push({ surface, target }); + } + + async inspectWriteTarget(target: string): Promise { + const explicit = this.states.get(target); + if (explicit !== undefined) { + return explicit; + } + const contents = this.files.get(target); + return contents === undefined + ? { kind: "missing", hasSymlinkAncestor: false } + : { kind: "file", contents, hasSymlinkAncestor: false }; + } + + async prepareRuntime(): Promise { + this.mutations.push({ kind: "prepare-runtime" }); + if (this.failRuntimePreparation) { + throw new Error("injected runtime preparation failure"); + } + } + + async writeFileAtomic(target: string, contents: string): Promise { + this.mutations.push({ kind: "write", target }); + if (target === this.failWriteTarget) { + throw new Error("injected atomic write failure"); + } + this.files.set(target, contents); + } +} + +test("the shipped skill resources preserve routing and companion contracts", async () => { + const repositoryRoot = resolve(import.meta.dir, ".."); + const [ + skill, + customize, + companionSkills, + template, + percolate, + design, + readme, + cli, + index, + manifestSource, + skillFiles, + ] = await Promise.all([ + readFile(resolve(repositoryRoot, "skills/kb/SKILL.md"), "utf8"), + readFile(resolve(repositoryRoot, "skills/kb/references/customize.md"), "utf8"), + readFile(resolve(repositoryRoot, "skills/kb/references/companion-skills.md"), "utf8"), + readFile(resolve(repositoryRoot, "skills/kb/templates/companion-skill.template.md"), "utf8"), + readFile(resolve(repositoryRoot, "skills/kb/references/percolate.md"), "utf8"), + readFile(resolve(repositoryRoot, "docs/design.md"), "utf8"), + readFile(resolve(repositoryRoot, "README.md"), "utf8"), + readFile(resolve(repositoryRoot, "src/cli.ts"), "utf8"), + readFile(resolve(repositoryRoot, "src/index.ts"), "utf8"), + readFile(resolve(repositoryRoot, "package.json"), "utf8"), + readdir(resolve(repositoryRoot, "skills/kb"), { recursive: true }), + ]); + const manifest = JSON.parse(manifestSource) as { + readonly exports?: unknown; + readonly files?: unknown; + readonly version?: unknown; + }; + if ( + !Array.isArray(manifest.files) + || manifest.files.some((file) => typeof file !== "string") + ) { + throw new Error("package files must be an array of strings"); + } + const manifestFiles = manifest.files as string[]; + const publicSourceFiles = manifestFiles + .filter((file) => file.startsWith("src/") && file.endsWith(".ts")) + .toSorted(); + const publicSource = (await Promise.all( + publicSourceFiles.map(async (file) => + `${file}\0${await readFile(resolve(repositoryRoot, file), "utf8")}` + ), + )).join("\n"); + const resources = { + skill, + customize, + companionSkills, + template, + percolate, + design, + readme, + publicSource, + }; + + expect(validateKbSkillContractResources(resources)).toEqual([]); + + expect(validateKbSkillContractResources({ + ...resources, + skill: skill.replace("## Route the request", "## Request routing"), + })).toContain("SKILL.md must route requests before runtime preparation"); + for (const forbiddenPublicSource of [ + "export type LifecycleRole = \"plan\";", + "export type MemoryRole = \"plan\";", + "export function resolveLifecycleRole(): void {}", + "export function resolveMemoryRole(): void {}", + "const lifecycleOption = \"--role\";", + "const explicitLifecycleOption = \"--lifecycle-role\";", + ]) { + expect(validateKbSkillContractResources({ + ...resources, + publicSource: `${publicSource}\n${forbiddenPublicSource}`, + }).some((error) => error.startsWith("public package source must not expose"))) + .toBe(true); + } + + expect(skillFiles.map(String).toSorted()).toEqual([ + "AGENTS.md", + "SKILL.md", + "agents/openai.yaml", + "references/companion-skills.md", + "references/customize.md", + "references/pdf-review.md", + "references/percolate.md", + "references/plan-structure.md", + "references/plan.md", + "references/query.md", + "references/refresh.md", + "references/save-pdf.md", + "references/save-url.md", + "references/url-authentication.md", + "references/url-platforms.md", + "templates/companion-skill.template.md", + ]); + expect(manifest.version).toBe("0.18.0"); + expect(manifestFiles).toContain("skills/kb"); + expect(publicSourceFiles).toContain("src/repository-memory.ts"); + expect(Object.keys(manifest.exports as Record).toSorted()).toEqual([ + ".", + "./agent-context", + "./agent-guide-audit", + "./attachments", + "./authoring", + "./benchmark", + "./browser-profiles", + "./capture", + "./cli", + "./clip/acquire", + "./clip/args", + "./clip/bounded-byte-buffer", + "./clip/bundle-reader", + "./clip/cli", + "./clip/cookies", + "./clip/doctor", + "./clip/jobs", + "./clip/network", + "./clip/network-proxy", + "./clip/persist", + "./clip/refresh", + "./clip/terminal", + "./evaluation", + "./evaluation-builder", + "./evaluation-kb", + "./git", + "./graph", + "./navigation", + "./pdf", + "./percolate", + "./portfolio", + "./query", + "./repository-memory", + "./sdk", + "./search", + "./search-rules", + "./semantic", + "./source-inbox", + "./untrusted-content", + "./url-intelligence", + "./workflow", + "./workflows", + "./workflows/decision-context", + "./workflows/explain-change", + "./workflows/plan-radar", + ]); + expect(index.trim().split("\n")).toEqual([ + 'export * from "./agent-context.js";', + 'export * from "./agent-guide-audit.js";', + 'export * from "./authoring.js";', + 'export * from "./attachments.js";', + 'export * from "./benchmark.js";', + 'export * from "./evaluation.js";', + 'export * from "./evaluation-kb.js";', + 'export * from "./git.js";', + 'export * from "./graph.js";', + 'export * from "./init.js";', + 'export * from "./navigation.js";', + 'export * from "./oh-adoption.js";', + 'export * from "./percolate.js";', + 'export * from "./query.js";', + 'export * from "./repository-memory.js";', + 'export * from "./search.js";', + 'export * from "./semantic.js";', + 'export * from "./sdk.js";', + 'export * from "./source-inbox.js";', + 'export * from "./vault.js";', + 'export * from "./workflow.js";', + ]); + const usage = /export const usage = `([\s\S]*?)`;/u.exec(cli)?.[1] ?? ""; + expect(createHash("sha256").update(usage).digest("hex")) + .toBe("4e3c1e971eeef76a4b7479466914480162c1f3f2b6ab3ad2babcbd121a668576"); + const commandIdentities = usage + .split("\n") + .filter((line) => line.startsWith(" kb ")) + .map((line) => { + const tokens = line.trim().split(/\s+/u); + const command = tokens[1] ?? ""; + const action = tokens[2] ?? ""; + return /^[a-z][a-z-]*$/u.test(action) ? `${command} ${action}` : command; + }) + .toSorted(); + expect(commandIdentities).toEqual([ + "adapters", + "agents audit", + "agents check", + "agents identity", + "backlinks", + "capture diff", + "capture show", + "capture verify", + "catalog", + "check", + "clip", + "context", + "doctor", + "evaluate", + "graph", + "history", + "history search", + "inbox", + "index", + "init", + "inspect", + "links", + "list", + "note create", + "pdf", + "percolate", + "portfolio audit", + "portfolio search", + "refresh", + "relation add", + "relation list", + "relation remove", + "search", + "url-metadata backfill", + "url-metadata tool", + ]); +}); + +test("denial and no reply produce no mutation attempts", async () => { + for (const approval of [{ kind: "denied" }, { kind: "unanswered" }] as const) { + const capabilities = new FakeCapabilities(); + const result = await executeCustomizationContract( + proposal(), + approval, + capabilities, + ); + expect(result.status).toBe( + approval.kind === "denied" ? "denied" : "awaiting-approval", + ); + expect(result.runtimePreparation).toBe("not-needed"); + expect(capabilities.inspections).toEqual([]); + expect(capabilities.mutations).toEqual([]); + } +}); + +test("preapproval inspection instruments every surface without mutation", async () => { + const requested = proposal({ + reads: [ + { surface: "filesystem", target: "/repo/kb" }, + { surface: "repository", target: "/repo" }, + { surface: "application", target: "editor" }, + { surface: "account", target: "signed-in-profile" }, + { surface: "network", target: "https://example.com/source" }, + { surface: "integration", target: "capture-adapter" }, + ], + }); + const capabilities = new FakeCapabilities(); + const result = await inspectCustomizationContract(requested, capabilities); + + expect(result.status).toBe("inspected"); + expect(capabilities.inspections.map(({ surface }) => surface)).toEqual([ + "filesystem", + "repository", + "application", + "account", + "network", + "integration", + ]); + expect(capabilities.mutations).toEqual([]); +}); + +test("a changed proposal requires renewed approval", async () => { + const approved = proposal(); + const changed = proposal({ + writes: [{ + surface: "filesystem", + target: "different/SKILL.md", + contents: "# Different\n", + }], + }); + const capabilities = new FakeCapabilities(); + const result = await executeCustomizationContract( + changed, + { kind: "approved", proposalDigest: customizationProposalDigest(approved) }, + capabilities, + ); + + expect(result.status).toBe("needs-reapproval"); + expect(capabilities.mutations).toEqual([]); +}); + +test("approval writes only exact targets and an exact repeat is a no-op", async () => { + const requested = proposal({ runtime: "kb-cli" }); + const capabilities = new FakeCapabilities(); + const approval = { + kind: "approved" as const, + proposalDigest: customizationProposalDigest(requested), + }; + + const first = await executeCustomizationContract(requested, approval, capabilities); + expect(first).toMatchObject({ + status: "applied", + runtimePreparation: "completed", + changed: ["/approved/skills/save-decision-kb/SKILL.md"], + }); + expect(capabilities.mutations).toEqual([ + { kind: "prepare-runtime" }, + { kind: "write", target: "/approved/skills/save-decision-kb/SKILL.md" }, + ]); + + capabilities.mutations.length = 0; + const repeated = await executeCustomizationContract(requested, approval, capabilities); + expect(repeated).toMatchObject({ + status: "no-op", + runtimePreparation: "not-needed", + changed: [], + }); + expect(capabilities.mutations).toEqual([]); +}); + +test("divergent content stops without overwrite", async () => { + const requested = proposal(); + const capabilities = new FakeCapabilities(); + capabilities.files.set( + "/approved/skills/save-decision-kb/SKILL.md", + "# Existing divergent skill\n", + ); + const result = await executeCustomizationContract( + requested, + { kind: "approved", proposalDigest: customizationProposalDigest(requested) }, + capabilities, + ); + + expect(result).toMatchObject({ status: "rejected" }); + expect(capabilities.mutations).toEqual([]); + expect(capabilities.files.get("/approved/skills/save-decision-kb/SKILL.md")) + .toBe("# Existing divergent skill\n"); +}); + +test("path escape and symbolic links fail before mutation", async () => { + const escaped = proposal({ + writes: [{ + surface: "filesystem", + target: "../outside/SKILL.md", + contents: "# Escape\n", + }], + }); + const escapedCapabilities = new FakeCapabilities(); + expect((await executeCustomizationContract( + escaped, + { kind: "approved", proposalDigest: customizationProposalDigest(escaped) }, + escapedCapabilities, + )).status).toBe("rejected"); + expect(escapedCapabilities.mutations).toEqual([]); + + const linked = proposal(); + const linkedCapabilities = new FakeCapabilities(); + linkedCapabilities.states.set( + "/approved/skills/save-decision-kb/SKILL.md", + { kind: "missing", hasSymlinkAncestor: true }, + ); + expect((await executeCustomizationContract( + linked, + { kind: "approved", proposalDigest: customizationProposalDigest(linked) }, + linkedCapabilities, + )).status).toBe("rejected"); + expect(linkedCapabilities.mutations).toEqual([]); +}); + +test("portable path aliases fail before inspection or mutation", async () => { + for (const [firstTarget, secondTarget] of [ + ["Save-Decision-KB/SKILL.md", "save-decision-kb/skill.md"], + ["caf\u00e9/SKILL.md", "cafe\u0301/SKILL.md"], + ] as const) { + const requested = proposal({ + writes: [ + { surface: "filesystem", target: firstTarget, contents: "# First\n" }, + { surface: "filesystem", target: secondTarget, contents: "# Second\n" }, + ], + }); + const capabilities = new FakeCapabilities(); + const result = await executeCustomizationContract( + requested, + { kind: "approved", proposalDigest: customizationProposalDigest(requested) }, + capabilities, + ); + + expect(result).toMatchObject({ + status: "rejected", + runtimePreparation: "not-needed", + }); + expect(result.status === "rejected" ? result.reason : "") + .toContain("portable path normalization"); + expect(capabilities.mutations).toEqual([]); + } +}); + +test("runtime preparation failure is explicit and stops before file writes", async () => { + const requested = proposal({ runtime: "kb-cli" }); + const capabilities = new FakeCapabilities(); + capabilities.failRuntimePreparation = true; + const result = await executeCustomizationContract( + requested, + { kind: "approved", proposalDigest: customizationProposalDigest(requested) }, + capabilities, + ); + + expect(result).toMatchObject({ + status: "rejected", + runtimePreparation: "failed", + reason: "injected runtime preparation failure", + }); + expect(capabilities.mutations).toEqual([{ kind: "prepare-runtime" }]); +}); + +test("a mid-write failure is reported once without retry or rollback", async () => { + const requested = proposal({ + runtime: "kb-cli", + writes: [ + { + surface: "filesystem", + target: "save-decision-kb/SKILL.md", + contents: "# Save a decision\n", + }, + { + surface: "filesystem", + target: "save-decision-kb/reference.md", + contents: "# Reference\n", + }, + ], + }); + const capabilities = new FakeCapabilities(); + capabilities.failWriteTarget = "/approved/skills/save-decision-kb/reference.md"; + const result = await executeCustomizationContract( + requested, + { kind: "approved", proposalDigest: customizationProposalDigest(requested) }, + capabilities, + ); + + expect(result).toMatchObject({ + status: "partial", + runtimePreparation: "completed", + changed: ["/approved/skills/save-decision-kb/SKILL.md"], + failedTarget: "/approved/skills/save-decision-kb/reference.md", + }); + expect(capabilities.mutations).toEqual([ + { kind: "prepare-runtime" }, + { kind: "write", target: "/approved/skills/save-decision-kb/SKILL.md" }, + { kind: "write", target: "/approved/skills/save-decision-kb/reference.md" }, + ]); + expect(capabilities.files.has("/approved/skills/save-decision-kb/SKILL.md")).toBe(true); + expect(capabilities.files.has("/approved/skills/save-decision-kb/reference.md")).toBe(false); +}); + +test("non-filesystem write surfaces are never treated as approved scaffolding", async () => { + for (const surface of [ + "repository", + "application", + "account", + "network", + "integration", + ] as const) { + const requested = proposal({ + writes: [{ + surface, + target: `${surface}/target`, + contents: "enabled=true\n", + }], + }); + const capabilities = new FakeCapabilities(); + const result = await executeCustomizationContract( + requested, + { kind: "approved", proposalDigest: customizationProposalDigest(requested) }, + capabilities, + ); + + expect(result).toMatchObject({ + status: "rejected", + runtimePreparation: "not-needed", + }); + expect(result.status === "rejected" ? result.reason : "") + .toContain(JSON.stringify(surface)); + expect(capabilities.mutations).toEqual([]); + } +}); diff --git a/scripts/kb-skill-contract.ts b/scripts/kb-skill-contract.ts new file mode 100644 index 0000000..cfde95a --- /dev/null +++ b/scripts/kb-skill-contract.ts @@ -0,0 +1,486 @@ +import { createHash } from "node:crypto"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +export type CustomizationSurface = + | "filesystem" + | "repository" + | "application" + | "account" + | "network" + | "integration"; + +export type CustomizationProposal = { + readonly id: string; + readonly root: string; + readonly runtime: "none" | "kb-cli"; + readonly reads: readonly { + readonly surface: CustomizationSurface; + readonly target: string; + }[]; + readonly writes: readonly { + readonly surface: CustomizationSurface; + readonly target: string; + readonly contents: string; + }[]; +}; + +export type CustomizationApproval = + | { readonly kind: "approved"; readonly proposalDigest: string } + | { readonly kind: "denied" } + | { readonly kind: "unanswered" }; + +export type CustomizationTargetState = + | { readonly kind: "missing"; readonly hasSymlinkAncestor: boolean } + | { + readonly kind: "file"; + readonly contents: string; + readonly hasSymlinkAncestor: boolean; + } + | { readonly kind: "symlink"; readonly hasSymlinkAncestor: boolean } + | { readonly kind: "other"; readonly hasSymlinkAncestor: boolean }; + +export type CustomizationCapabilities = { + readonly inspect: ( + surface: CustomizationSurface, + target: string, + ) => Promise; + readonly inspectWriteTarget: ( + absolutePath: string, + ) => Promise; + readonly prepareRuntime: () => Promise; + readonly writeFileAtomic: ( + absolutePath: string, + contents: string, + ) => Promise; +}; + +export type RuntimePreparationStatus = "not-needed" | "completed" | "failed"; + +type CustomizationExecutionTrace = { + readonly proposalDigest: string; + readonly runtimePreparation: RuntimePreparationStatus; +}; + +export type CustomizationExecutionResult = CustomizationExecutionTrace & ( + | { readonly status: "awaiting-approval" } + | { readonly status: "denied" } + | { readonly status: "needs-reapproval" } + | { + readonly status: "rejected"; + readonly reason: string; + } + | { + readonly status: "no-op" | "applied"; + readonly changed: readonly string[]; + } + | { + readonly status: "partial"; + readonly changed: readonly string[]; + readonly failedTarget: string; + readonly reason: string; + } +); + +export type CustomizationInspectionResult = + | { readonly status: "inspected"; readonly proposalDigest: string } + | { + readonly status: "rejected"; + readonly proposalDigest: string; + readonly reason: string; + }; + +type PreparedWrite = { + readonly absolutePath: string; + readonly contents: string; + readonly state: CustomizationTargetState; +}; + +const MAX_INSPECTIONS = 64; +const MAX_WRITES = 16; +const MAX_CONTENT_BYTES = 1024 * 1024; + +function canonicalProposal(proposal: CustomizationProposal): string { + return JSON.stringify({ + id: proposal.id, + root: proposal.root, + runtime: proposal.runtime, + reads: proposal.reads.map(({ surface, target }) => ({ surface, target })), + writes: proposal.writes.map(({ surface, target, contents }) => ({ + surface, + target, + contents, + })), + }); +} + +export function customizationProposalDigest( + proposal: CustomizationProposal, +): string { + return createHash("sha256").update(canonicalProposal(proposal)).digest("hex"); +} + +export async function inspectCustomizationContract( + proposal: CustomizationProposal, + capabilities: Pick, +): Promise { + const proposalDigest = customizationProposalDigest(proposal); + if (proposal.reads.length > MAX_INSPECTIONS) { + return { + status: "rejected", + proposalDigest, + reason: `proposal exceeds ${MAX_INSPECTIONS} read surfaces`, + }; + } + for (const read of proposal.reads) { + await capabilities.inspect(read.surface, read.target); + } + return { status: "inspected", proposalDigest }; +} + +function confinedTarget(root: string, target: string): string | null { + if (!isAbsolute(root) || isAbsolute(target) || target.length === 0) { + return null; + } + const absolutePath = resolve(root, target); + const fromRoot = relative(root, absolutePath); + if ( + fromRoot.length === 0 + || fromRoot === ".." + || fromRoot.startsWith(`..${sep}`) + || isAbsolute(fromRoot) + ) { + return null; + } + return absolutePath; +} + +function portableTargetIdentity(absolutePath: string): string { + return absolutePath.normalize("NFC").toLocaleLowerCase("en-US"); +} + +function validateProposalShape(proposal: CustomizationProposal): string | null { + if (proposal.id.trim().length === 0) { + return "proposal id is required"; + } + if (proposal.reads.length > MAX_INSPECTIONS) { + return `proposal exceeds ${MAX_INSPECTIONS} read surfaces`; + } + if (proposal.writes.length > MAX_WRITES) { + return `proposal exceeds ${MAX_WRITES} write targets`; + } + const targets = new Set(); + let contentBytes = 0; + for (const write of proposal.writes) { + if (write.surface !== "filesystem") { + return `write surface ${JSON.stringify(write.surface)} is not an approved scaffold surface`; + } + const absolutePath = confinedTarget(proposal.root, write.target); + if (absolutePath === null) { + return `write target ${JSON.stringify(write.target)} escapes the approved root`; + } + const targetIdentity = portableTargetIdentity(absolutePath); + if (targets.has(targetIdentity)) { + return `write target ${JSON.stringify(write.target)} aliases another scaffold target after portable path normalization`; + } + targets.add(targetIdentity); + contentBytes += Buffer.byteLength(write.contents, "utf8"); + if (contentBytes > MAX_CONTENT_BYTES) { + return `proposal exceeds ${MAX_CONTENT_BYTES} bytes of durable output`; + } + } + return null; +} + +export async function executeCustomizationContract( + proposal: CustomizationProposal, + approval: CustomizationApproval, + capabilities: CustomizationCapabilities, +): Promise { + const proposalDigest = customizationProposalDigest(proposal); + const noRuntime = { + proposalDigest, + runtimePreparation: "not-needed" as const, + }; + + if (approval.kind === "unanswered") { + return { status: "awaiting-approval", ...noRuntime }; + } + if (approval.kind === "denied") { + return { status: "denied", ...noRuntime }; + } + if (approval.proposalDigest !== proposalDigest) { + return { status: "needs-reapproval", ...noRuntime }; + } + + const shapeError = validateProposalShape(proposal); + if (shapeError !== null) { + return { status: "rejected", ...noRuntime, reason: shapeError }; + } + + const prepared: PreparedWrite[] = []; + for (const write of proposal.writes) { + const absolutePath = confinedTarget(proposal.root, write.target); + if (absolutePath === null) { + return { + status: "rejected", + ...noRuntime, + reason: `write target ${JSON.stringify(write.target)} escapes the approved root`, + }; + } + const state = await capabilities.inspectWriteTarget(absolutePath); + if (state.hasSymlinkAncestor || state.kind === "symlink") { + return { + status: "rejected", + ...noRuntime, + reason: `write target ${JSON.stringify(write.target)} crosses a symbolic link`, + }; + } + if (state.kind === "other") { + return { + status: "rejected", + ...noRuntime, + reason: `write target ${JSON.stringify(write.target)} is not a regular file`, + }; + } + if (state.kind === "file" && state.contents !== write.contents) { + return { + status: "rejected", + ...noRuntime, + reason: `write target ${JSON.stringify(write.target)} has divergent content`, + }; + } + prepared.push({ absolutePath, contents: write.contents, state }); + } + + const changed: string[] = []; + for (const write of prepared) { + if (write.state.kind !== "file") { + changed.push(write.absolutePath); + } + } + + if (changed.length === 0) { + return { + status: "no-op", + ...noRuntime, + changed: Object.freeze([]), + }; + } + + let runtimePreparation: RuntimePreparationStatus = "not-needed"; + if (proposal.runtime === "kb-cli") { + try { + await capabilities.prepareRuntime(); + runtimePreparation = "completed"; + } catch (error) { + return { + status: "rejected", + proposalDigest, + runtimePreparation: "failed", + reason: error instanceof Error ? error.message : String(error), + }; + } + } + + const completed: string[] = []; + for (const write of prepared) { + if (write.state.kind === "file") { + continue; + } + try { + await capabilities.writeFileAtomic(write.absolutePath, write.contents); + completed.push(write.absolutePath); + } catch (error) { + return { + status: "partial", + proposalDigest, + runtimePreparation, + changed: Object.freeze([...completed]), + failedTarget: write.absolutePath, + reason: error instanceof Error ? error.message : String(error), + }; + } + } + + return { + status: "applied", + proposalDigest, + runtimePreparation, + changed: Object.freeze(completed), + }; +} + +export type KbSkillContractResources = { + readonly skill: string; + readonly customize: string; + readonly companionSkills: string; + readonly template: string; + readonly percolate: string; + readonly design: string; + readonly readme: string; + readonly publicSource: string; +}; + +const CUSTOMIZE_HEADINGS = [ + "Customize a KB setup", + "Establish the boundary", + "Inspect without mutation", + "Interview in small batches", + "Propose the smallest useful change", + "Obtain approval", + "Scaffold within the approved boundary", + "Start with real material", + "Verify and hand off", + "Evolve an existing setup", +] as const; + +const COMPANION_HEADINGS = [ + "Companion skill contracts", + "Identity and routing", + "Inputs and preconditions", + "Surfaces and authority", + "Approval boundary", + "Execution semantics", + "Durable outputs and provenance", + "Verification and KB maintenance", + "Composition boundary", + "Review checklist", +] as const; + +const TEMPLATE_HEADINGS = [ + "Use when", + "Do not use when", + "Inputs and preconditions", + "Surfaces and authority", + "Approval", + "Workflow", + "Idempotence, retries, and failure", + "Durable outputs and provenance", + "Verification", +] as const; + +function missingHeadings(contents: string, headings: readonly string[]): string[] { + const actual = new Set(Array.from( + contents.matchAll(/^#{1,6}\s+(.+?)\s*$/gmu), + (match) => match[1] ?? "", + )); + return headings.filter((heading) => !actual.has(heading)); +} + +function templateFrontmatterKeys(contents: string): string[] | null { + const lines = contents.split("\n"); + if (lines[0] !== "---") { + return null; + } + const end = lines.indexOf("---", 1); + if (end < 0) { + return null; + } + const keys: string[] = []; + for (const line of lines.slice(1, end)) { + if (line.trim().length === 0) { + continue; + } + const match = /^([a-z][a-z0-9_-]*):(?:\s|$)/u.exec(line); + if (match?.[1] === undefined) { + return null; + } + keys.push(match[1]); + } + return keys; +} + +export function validateKbSkillContractResources( + resources: KbSkillContractResources, +): readonly string[] { + const errors: string[] = []; + const routeIndex = resources.skill.indexOf("## Route the request"); + const runtimeIndex = resources.skill.indexOf("## Prepare the runtime"); + if (routeIndex < 0 || runtimeIndex < 0 || routeIndex >= runtimeIndex) { + errors.push("SKILL.md must route requests before runtime preparation"); + } + for (const link of [ + "references/customize.md", + "references/companion-skills.md", + ]) { + if (!resources.skill.includes(link)) { + errors.push(`SKILL.md must link ${link}`); + } + } + if (!resources.skill.includes("bun add --global @hraness/kb@0.18.0")) { + errors.push("SKILL.md must retain the immutable 0.18.0 runtime pin"); + } + for (const [name, contents, headings] of [ + ["customize.md", resources.customize, CUSTOMIZE_HEADINGS], + ["companion-skills.md", resources.companionSkills, COMPANION_HEADINGS], + ["companion-skill.template.md", resources.template, TEMPLATE_HEADINGS], + ] as const) { + for (const heading of missingHeadings(contents, headings)) { + errors.push(`${name} is missing heading ${JSON.stringify(heading)}`); + } + } + const frontmatterKeys = templateFrontmatterKeys(resources.template); + if ( + frontmatterKeys === null + || frontmatterKeys.length !== 2 + || frontmatterKeys[0] !== "name" + || frontmatterKeys[1] !== "description" + ) { + errors.push("companion skill template frontmatter must contain only name and description"); + } + for (const required of [ + "Silence, a denial, or an ambiguous response is not approval.", + "//SKILL.md", + "Do not run `kb doctor`, `kb init`, `kb index`, QMD", + "The scaffold executor writes filesystem targets only.", + ]) { + if (!resources.customize.includes(required)) { + errors.push(`customize.md must include ${JSON.stringify(required)}`); + } + } + for (const required of [ + "available through the 0.18 release line", + "not removed before 0.19.0", + "any canonical custom predicate", + "Never write inverse edges", + "https://gist.github.com/fxchen/773397095d7a6bffda621e4237da0da9", + "https://gist.github.com/fxchen/09cb410b22c9c5256d80243ee925b57e", + ]) { + if (!resources.percolate.includes(required)) { + errors.push(`percolate.md must include ${JSON.stringify(required)}`); + } + } + for (const required of [ + "ships no `kb_role` field", + "lifecycle resolver or API", + "lifecycle CLI", + ]) { + if (!resources.design.includes(required) && !resources.readme.includes(required)) { + errors.push(`public documentation must include ${JSON.stringify(required)}`); + } + } + for (const { label, pattern } of [ + { label: "kb_role", pattern: /\bkb_role\b/iu }, + { + label: "a lifecycle-role identity", + pattern: + /\b(?:KbLifecycle|Lifecycle|Memory|Document|Knowledge|Authority)Role(?:Resolver|Resolution)?\b/iu, + }, + { + label: "a lifecycle-role resolver", + pattern: + /\b(?:resolve|classify|infer|assign|audit|inspect)(?:KbLifecycle|Lifecycle|Memory|Document|Knowledge|Authority)Role\b/iu, + }, + { + label: "a lifecycle-role CLI option", + pattern: + /--(?:kb-)?(?:lifecycle-|memory-|document-|knowledge-|authority-)?role\b/iu, + }, + { label: "kb lifecycle", pattern: /\bkb[ -]lifecycle\b/iu }, + ]) { + if (pattern.test(resources.publicSource)) { + errors.push(`public package source must not expose ${label}`); + } + } + return Object.freeze(errors); +} diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index ebb45f4..37b4d12 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -60,6 +60,7 @@ const importSpecifiers = [ "@hraness/kb/workflows/plan-radar", ]; const requiredNamedExports = { + "@hraness/kb": ["createOhAdoptionPreparerV1"], "@hraness/kb/clip/bundle-reader": ["readCaptureBundle", "verifyCaptureBundle"], "@hraness/kb/clip/jobs": ["createCaptureJob", "openCaptureJobStore", "updateCaptureJob"], "@hraness/kb/clip/refresh": ["diffCaptureBundle"], @@ -625,6 +626,7 @@ for (const specifier of ${JSON.stringify(importSpecifiers)}) { const consumerSource = `${importSpecifiers.map((specifier, index) => `import * as surface${String(index)} from ${JSON.stringify(specifier)};` ).join("\n")} +import { createOhAdoptionPreparerV1 } from "@hraness/kb"; import { readCaptureBundle, verifyCaptureBundle } from "@hraness/kb/clip/bundle-reader"; import { createCaptureJob, openCaptureJobStore, updateCaptureJob } from "@hraness/kb/clip/jobs"; import { diffCaptureBundle } from "@hraness/kb/clip/refresh"; @@ -644,6 +646,7 @@ const registry = parsePortfolioRegistry({ const identity = parseQualifiedDocumentUri("kb://hraness/kb/note-id"); const projected = projectUntrustedJson([{ title: "stored source" }]); void [ + createOhAdoptionPreparerV1, readCaptureBundle, verifyCaptureBundle, createCaptureJob, openCaptureJobStore, updateCaptureJob, diffCaptureBundle, openKnowledgePortfolio, prioritizeSearchHits, diff --git a/skills/kb/AGENTS.md b/skills/kb/AGENTS.md index ef30f9d..e5dea0a 100644 --- a/skills/kb/AGENTS.md +++ b/skills/kb/AGENTS.md @@ -2,10 +2,12 @@ - `SKILL.md` – public entrypoint, runtime contract, and intent router for all hraness/kb agent workflows. - `references/query.md` – scoped retrieval through exact metadata, hybrid search, graph structure, and Git provenance. +- `references/customize.md` and `companion-skills.md` – interview-first KB setup, explicit approval boundaries, and contracts for optional recurring rituals. - `references/save-url.md`, `url-authentication.md`, and `url-platforms.md` – auditable web capture, signed-in source handling, and platform-specific completeness boundaries. - `references/save-pdf.md` and `pdf-review.md` – PDF ingestion, OCR, image evidence, and mixed-media review. - `references/plan.md` and `plan-structure.md` – durable plan authoring and its adaptable Markdown contract. - `references/percolate.md` and `refresh.md` – evidence-backed graph edits, catalog maintenance, and vault validation. +- `templates/companion-skill.template.md` – inert, copyable starting point for an approved companion skill. - `agents/openai.yaml` – user-facing skill metadata and invocation prompt. # Guidelines @@ -14,6 +16,7 @@ - Keep discovery language grounded in real user requests for knowledge bases, coding-agent memory, Markdown or Obsidian vaults, source capture, repository context, plans, and knowledge graphs. Exclude generic research, PDF reading, and planning outside hraness/kb. - Invoke the installed `kb` CLI without depending on a source checkout. Check for an existing command first, require Bun when installation is needed, and pin installation to the current immutable repository tag. - Never initialize or mutate a vault as part of skill or CLI installation. +- Route setup, evolution, and custom-ritual requests before runtime discovery. Keep inspection non-mutating until the exact proposal is approved, and never treat skill discovery or an ambient account as authority. - Keep Markdown authoritative. Preserve incomplete source boundaries, read cited evidence before graph edits, and never generate reciprocal, inferred, transitive, or similarity-derived relationships. - In parallel managed-catalog work, defer refresh to the integrating agent and use the catalog-skipping check in each edit lane. Authored-catalog refreshes leave the front door unchanged. - Update `agents/openai.yaml`, README installation text, package inventory checks, and the pinned CLI tag together when the skill identity or package release changes. diff --git a/skills/kb/SKILL.md b/skills/kb/SKILL.md index 85d3078..3acd545 100644 --- a/skills/kb/SKILL.md +++ b/skills/kb/SKILL.md @@ -1,14 +1,15 @@ --- name: kb description: >- - Operate a hraness/kb local-first Markdown knowledge base for coding-agent - memory. Use when a user asks to search or query a KB or Obsidian vault; load - repository context, plans, decisions, concepts, backlinks, semantic search, - or Git provenance; save, clip, scrape, or archive a URL, article, social - thread, signed-in browser page, or PDF as auditable Markdown; create or - update a durable plan in the vault; or refresh, check, percolate, and - maintain its knowledge graph. Do not use for generic web research, generic - PDF reading, or ordinary planning that will not use a hraness/kb vault. + Set up, evolve, or operate a hraness/kb local-first Markdown knowledge base + for coding-agent memory. Use when a user asks to design KB conventions or a + recurring KB ritual; search or query a KB or Obsidian vault; load repository + context, plans, decisions, concepts, backlinks, semantic search, or Git + provenance; save, clip, scrape, or archive a URL, article, social thread, + signed-in browser page, or PDF as auditable Markdown; create or update a + durable plan in the vault; or refresh, check, percolate, and maintain its + knowledge graph. Do not use for generic web research, generic PDF reading, + or ordinary planning that will not use a hraness/kb vault. --- # Work with KB @@ -17,7 +18,29 @@ Use hraness/kb to preserve and retrieve inspectable agent memory in Markdown and Git. Select the smallest workflow that matches the request, then load only its references. -## Prepare the runtime +## Route the request + +Route the request before discovering, installing, or running the CLI. A setup, +evolution, or custom-ritual request begins with read-only inspection and an +approved proposal; it does not require a runtime merely because this skill was +selected. + +| User intent | Read | +| --- | --- | +| Design, set up, or evolve a KB; choose its boundaries and conventions; or define a recurring KB ritual | [Customize a KB setup](references/customize.md); add [Companion skill contracts](references/companion-skills.md) only when the proposal includes a new or revised skill | +| Find notes, search one vault or an authorized portfolio, load repository-path context, inspect plans or decisions, follow backlinks or relationships, audit vault organization, or retrieve Git provenance | [Query the knowledge base](references/query.md) | +| Save, clip, scrape, or archive a URL, article, social post or thread, GitHub or Discourse discussion, signed-in page, feed, inbox, private document, WhatsApp conversation, or YouTube page | [Capture web content](references/save-url.md); add [browser authentication](references/url-authentication.md) for signed-in sources and [platform routing](references/url-platforms.md) when route choice or completeness matters | +| Import, extract, archive, OCR, or convert a local or public PDF into Markdown | [Save a PDF](references/save-pdf.md); add [PDF image review](references/pdf-review.md) for scans, screenshots, conversations, charts, or mixed media | +| Create or update an implementation plan, proposal, RFC, migration plan, execution audit, or phased checklist in the vault | [Write a durable plan](references/plan.md) and [use its structure](references/plan-structure.md) | +| Review recurring ideas, promote concepts, or add and verify typed relationships | [Percolate concepts and relationships](references/percolate.md) | +| Refresh or validate the catalog, graph, attachments, repository scopes, context mappings, or overall vault health | [Refresh and check the knowledge base](references/refresh.md) | + +Read more than one primary reference only when the request spans those +workflows. For example, saving a source and linking it from a maintained note +uses the capture workflow followed by the relevant percolation and refresh +steps. + +## Prepare the runtime when execution needs it Use an existing `kb` command when one is available. Do not reinstall or upgrade it merely because this skill loaded. @@ -46,28 +69,13 @@ Installation ends after command verification. Never run `kb init`, create a vault, refresh a catalog, or edit Markdown as an installation side effect. Initialize or mutate a vault only when the user's request requires that change. -## Route the request - -| User intent | Read | -| --- | --- | -| Find notes, search one vault or an authorized portfolio, load repository-path context, inspect plans or decisions, follow backlinks or relationships, audit vault organization, or retrieve Git provenance | [Query the knowledge base](references/query.md) | -| Save, clip, scrape, or archive a URL, article, social post or thread, GitHub or Discourse discussion, signed-in page, feed, inbox, private document, WhatsApp conversation, or YouTube page | [Capture web content](references/save-url.md); add [browser authentication](references/url-authentication.md) for signed-in sources and [platform routing](references/url-platforms.md) when route choice or completeness matters | -| Import, extract, archive, OCR, or convert a local or public PDF into Markdown | [Save a PDF](references/save-pdf.md); add [PDF image review](references/pdf-review.md) for scans, screenshots, conversations, charts, or mixed media | -| Create or update an implementation plan, proposal, RFC, migration plan, execution audit, or phased checklist in the vault | [Write a durable plan](references/plan.md) and [use its structure](references/plan-structure.md) | -| Review recurring ideas, promote concepts, or add and verify typed relationships | [Percolate concepts and relationships](references/percolate.md) | -| Refresh or validate the catalog, graph, attachments, repository scopes, context mappings, or overall vault health | [Refresh and check the knowledge base](references/refresh.md) | - -Read more than one primary reference only when the request spans those -workflows. For example, saving a source and linking it from a maintained note -uses the capture workflow followed by the relevant percolation and refresh -steps. - ## Preserve the KB contract -- Resolve `KB_ROOT` to the existing vault directory that contains its managed - or authored `index.md`. Read the applicable repository and vault - `AGENTS.md` files before writing. Do not assume the session started in the - vault. +- For an existing vault, resolve `KB_ROOT` to the directory that contains its + managed or authored `index.md`. During setup, inspect the explicitly proposed + location without assuming that `index.md` or any KB directory exists. Read + the applicable repository and vault `AGENTS.md` files before writing. Do not + assume the session started in the vault. - Treat authored Markdown and Git as the record. Catalogs, backlinks, graph reports, search indexes, embeddings, and percolation candidates are derived views. diff --git a/skills/kb/agents/openai.yaml b/skills/kb/agents/openai.yaml index 928e6c9..5ff2bfd 100644 --- a/skills/kb/agents/openai.yaml +++ b/skills/kb/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "KB" - short_description: "Query, capture, plan, and maintain Markdown memory" - default_prompt: "Use $kb to work with this hraness/kb knowledge base and load only the workflow references needed for the request." + short_description: "Set up, query, capture, and maintain Markdown memory" + default_prompt: "Use $kb to set up or work with this hraness/kb knowledge base, routing the request before any runtime preparation and loading only the references needed." diff --git a/skills/kb/references/companion-skills.md b/skills/kb/references/companion-skills.md new file mode 100644 index 0000000..fc1a25d --- /dev/null +++ b/skills/kb/references/companion-skills.md @@ -0,0 +1,96 @@ +# Companion skill contracts + +A companion skill handles one recurring KB ritual that benefits from a +discriminating trigger and an explicit operating contract. It composes with +the public `kb` skill. It does not register code at runtime, execute vault +metadata, or gain authority by being installed. + +## Identity and routing + +Give the skill a lowercase action-oriented name and a description that states +the concrete request that should select it. Keep generic querying, capture, +planning, percolation, refresh, and validation in the main `kb` skill. Propose +at most three companions, and prefer zero when the standard router is enough. + +## Inputs and preconditions + +List the exact inputs that must be supplied or resolved before work starts. +Distinguish an existing vault from a proposed location. State required local +commands, repository state, source availability, and authorization without +installing or probing them as a side effect of skill discovery. + +## Surfaces and authority + +List each filesystem, repository, application, account, network, and +integration surface the workflow may read or write. Skill discovery, +installation, or an existing signed-in session grants no authority. The user's +scope, the applicable repository instructions, host permissions, and the +selected tool's own approval boundary remain controlling. + +Do not infer that an account operation is read-only from its HTTP method. Do +not place secrets, cookies, tokens, session data, or ambient personal context +in durable output. + +The shipped customization executor proves filesystem scaffolding only. It +does not execute an application, account, network, or integration write. A +companion skill that later needs such an action must treat it as a separate +runtime request with its own exact proposal, approval, capable tool, and +inspectable result. + +## Approval boundary + +Separate read-only inspection from mutation. Present exact targets and writes +before approval unless the user's request already authorizes them. A denial, +no response, changed proposal, path expansion, added account, or new external +surface requires stopping or renewed approval. + +## Execution semantics + +Define deterministic behavior for the first run and an exact repeat. Require +path confinement, reject symbolic-link targets, and preserve divergent +existing content. Name each effect explicitly instead of granting a broad +filesystem or application capability. + +## Durable outputs and provenance + +Name the files or records that persist, their authority, and the provenance +they retain. Markdown and Git remain authoritative KB state. Generated +catalogs, indexes, embeddings, and graph reports stay rebuildable. Exclude +credentials, session material, and unrelated account data. + +## Verification and KB maintenance + +Define the narrow checks that establish the intended result. After material KB +edits, review percolation candidates and run the appropriate catalog-aware +check. Parallel lanes use `kb check --no-catalog`; one integrating agent owns a +managed catalog refresh. + +## Composition boundary + +Call the installed `kb` command or its public package interfaces only when the +approved workflow needs them. Do not add a plugin registry, hook loader, +background process, executable vault metadata, or implicit account bridge. A +companion skill is an instruction boundary, not runtime extensibility. + +## Review checklist + +The repository's fake-capability suite is a tested contract example. It checks +the expected approval and failure transitions, but it does not prove that every +agent or host integration complies. Review the executing agent's actual tool +and permission boundaries as well. + +- Does the trigger identify one recurring request without attracting generic + KB work? +- Are inputs, preconditions, read surfaces, write surfaces, and exact targets + explicit? +- Can inspection finish without installing, indexing, caching, or mutating? +- Does approval cover every effect, with renewed approval for any change? +- Is an exact repeat a no-op, while divergence, path escape, symlinks, partial + failure, and unapproved external access stop safely? +- Do durable outputs preserve useful provenance without secrets or session + data? +- Does the skill compose with the public router without copying its general + instructions? + +Start from [`companion-skill.template.md`](../templates/companion-skill.template.md) +only after the proposal's target skill root and name are approved. diff --git a/skills/kb/references/customize.md b/skills/kb/references/customize.md new file mode 100644 index 0000000..198f909 --- /dev/null +++ b/skills/kb/references/customize.md @@ -0,0 +1,123 @@ +# Customize a KB setup + +Design the smallest KB arrangement that answers the user's recurring memory +questions. Begin with an interview and read-only inspection. Do not install a +runtime, initialize a vault, build an index, access an account, or write a file +before the proposal has the user's approval. + +## Establish the boundary + +Identify the repositories, vaults, people, agents, and time horizons in scope. +Ask what the KB must help a future agent recover, which information must remain +outside it, and which existing instructions govern the target paths. Treat the +user's explicit request as authorization for the named work. Do not extend it +to another path, repository, account, application, or integration. + +For a new vault, ask for or propose an exact location. Do not require an +existing `index.md`. For an existing vault, resolve its front door and read the +applicable `AGENTS.md` files before proposing changes. + +## Inspect without mutation + +Inspect only the surfaces needed to understand the current setup. Typical +evidence includes directory structure, existing Markdown conventions, scoped +agent guides, active plans, source records, repository paths, and available +local commands. Keep filesystem, application, account, network, and +integration access within the user's stated scope and the host's actual +permissions. + +Do not run `kb doctor`, `kb init`, `kb index`, QMD, hybrid or semantic search, +an installer, or a command that may create a cache during this phase. Do not +create a hidden profile such as `.context/me.md` or infer personal context from +an ambient account. + +## Interview in small batches + +Ask only questions whose answers change the proposed structure. Prefer a small +batch about one decision at a time: + +- Which recurring questions should the KB answer? +- Which sources, maintained explanations, plans, and repository rules already + exist? +- Which writes should happen automatically, require review, or never happen? +- Which recurring action is common enough to justify a companion skill? + +Summarize each resolved decision before moving to the next uncertain one. A +short interview may conclude that the standard profile or no change is best. + +## Propose the smallest useful change + +Describe the exact files and surfaces before editing. Use this table: + +| Surface | Exact target | Read | Write | Purpose | Approval | +| --- | --- | --- | --- | --- | --- | +| Vault | `` | `` | `` | `` | `` | + +Propose zero to three companion skills. Each proposed skill must own a distinct +recurring request that the main `kb` router cannot express clearly enough. Do +not add a skill only to restate repository policy or wrap one command. + +State the verification, idempotence, retry, and failure behavior for every +write. Keep Markdown and Git authoritative. Treat indexes, embeddings, +catalogs, graph reports, and caches as replaceable views. + +## Obtain approval + +Present the proposal and wait when its writes are not already authorized by +the user's explicit request. Approval applies to the exact targets and +operations shown. A changed path, expanded repository, additional skill, +account surface, network action, or broader write requires renewed approval. + +Silence, a denial, or an ambiguous response is not approval. Inspection does +not grant write authority. Discovery of a command, application, account, or +integration does not authorize its use. + +## Scaffold within the approved boundary + +Create only approved paths. For a companion skill, read [Companion skill +contracts](companion-skills.md) and copy +[`companion-skill.template.md`](../templates/companion-skill.template.md) to +`//SKILL.md`. Never edit the template inside an +installed package or `node_modules`. + +If approved execution needs the KB CLI, prepare the runtime now using the main +skill's pinned installation instructions. Installation does not authorize +`kb init`, indexing, semantic search, or vault writes. Run only the approved +commands and exact allowlisted writes. + +The scaffold executor writes filesystem targets only. It never performs an +application, account, network, or integration write. A companion skill may +describe one of those later actions, but its execution is a separate runtime +request with its own exact proposal, approval, tool boundary, and result. + +On a repeated request, compare the desired bytes with the approved targets. +Treat an exact match as a no-op. Stop on divergent existing content, a symlink, +a path that escapes the approved root, an unapproved external surface, or a +partial write. Report the retained state instead of overwriting, silently +retrying, or widening the boundary. + +## Start with real material + +Use a small amount of material that exercises the agreed structure: one saved +source, one maintained explanation, one plan, or one repository-context +mapping. Do not manufacture empty directories, placeholder notes, a complete +ontology, or speculative metadata merely to make the vault look populated. + +## Verify and hand off + +Verify every approved file and record the exact paths changed. Run the +narrowest applicable KB checks only when they were approved and the runtime is +available. State what remains unconfigured, which views are rebuildable, and +which action would require separate authority. + +Keep durable output free of credentials, session material, account exports, +and hidden ambient context. Record source provenance and the boundary of any +incomplete acquisition. + +## Evolve an existing setup + +Re-run the boundary, inspection, interview, proposal, and approval steps when +the vault's recurring questions change. Prefer a focused convention or skill +revision to a migration. Preserve authored Markdown and Git history, and do not +mass-rewrite metadata to fit a new taxonomy unless a measured retrieval or +maintenance problem justifies that work. diff --git a/skills/kb/references/percolate.md b/skills/kb/references/percolate.md index 49817ab..5d500c9 100644 --- a/skills/kb/references/percolate.md +++ b/skills/kb/references/percolate.md @@ -42,6 +42,16 @@ minimum of two therefore requires two shared signals, not merely both endpoints of one tag match. Other candidate kinds count their natural unit: supporting notes, mention occurrences, or authored hygiene evidence. +Percolation Result V2 reports a missing relationship as an unordered pair of +endpoints with `predicate: { "kind": "required" }`. The output does not choose +which note owns the assertion, its direction, or its predicate. In particular, +it never inserts `related-to` as a fallback. Read both notes and their evidence, +then choose a source, target, and predicate only when the prose establishes that +claim. Historical unversioned V1 results may contain a suggested predicate; +parse them through the explicit V1 compatibility surface and do not treat that +suggestion as an authored fact or silently upgrade it to V2. V1 remains +available through the 0.18 release line and is not removed before 0.19.0. + For a missing concept, use `suggestedId`. When `collidesWith` is non-null, the natural ID is already an ordinary note, so KB chooses an unoccupied `*-concept` ID. Read the occupied note before deciding whether to create the @@ -88,11 +98,26 @@ kb relation add notes/write-path supports notes/durable-agent-memory \ --root "$KB_ROOT" ``` -Use a specific lower-kebab-case predicate. A local target is an exact -vault-root note ID without `.md`. A reviewed cross-vault target is its stable -qualified identity, such as `kb://hraness/kb/document-id`; never use a checkout -path as cross-vault identity. Ground the assertion in nearby prose or evidence; -the frontmatter is an indexable statement, not a substitute for explanation. +Use a specific lower-kebab-case predicate. Recommended predicates for common +KB evidence and maintenance claims are: + +- `synthesizes` when the source combines and maintains conclusions from the + target material; +- `evidenced-by` when the target directly supports a claim in the source; +- `informed-by` when the target influenced the source without serving as its + direct evidence; +- `supersedes` when the source deliberately replaces the target as the current + account; +- `contradicts` when the source records a supported incompatible claim. + +This vocabulary is advisory. A vault may use any canonical custom predicate +whose meaning its prose establishes. Do not assign a recommended predicate by +directory, note type, shared tags, chronology, or similarity alone. A local +target is an exact vault-root note ID without `.md`. A reviewed cross-vault +target is its stable qualified identity, such as +`kb://hraness/kb/document-id`; never use a checkout path as cross-vault +identity. Ground the assertion in nearby prose or evidence; the frontmatter is +an indexable statement, not a substitute for explanation. List or remove relationships without editing reciprocal notes: @@ -103,8 +128,15 @@ kb relation remove notes/write-path supports notes/durable-agent-memory \ ``` Never write inverse edges, generated backlinks, inferred transitive -relationships, or semantic-search scores into Markdown. Those are derived -views. +relationships, reciprocal edges, similarity-derived relationships, or +semantic-search scores into Markdown. External or unclassified material is +outside this vocabulary evaluation and remains unresolved. Those are derived +views or review work. + +The interview-first setup and relationship-review pattern builds on Frank +Chen's public notes about [designing a personal knowledge base with an +agent](https://gist.github.com/fxchen/773397095d7a6bffda621e4237da0da9) +and [extending it with skills](https://gist.github.com/fxchen/09cb410b22c9c5256d80243ee925b57e). ## Query before concluding diff --git a/skills/kb/templates/companion-skill.template.md b/skills/kb/templates/companion-skill.template.md new file mode 100644 index 0000000..87c3457 --- /dev/null +++ b/skills/kb/templates/companion-skill.template.md @@ -0,0 +1,57 @@ +--- +name: replace-with-skill-name +description: Replace with the recurring KB request that should select this skill. +--- + +# Replace with the skill title + +## Use when + +State the exact recurring KB request that this skill owns. + +## Do not use when + +Route generic query, capture, plan, percolation, refresh, and validation work +to the public `kb` skill. State any additional exclusions that prevent an +unsafe or ambiguous match. + +## Inputs and preconditions + +List required inputs, existing state, commands, and authorization. Do not +install, probe an account, create a cache, or mutate state while resolving +these preconditions. + +## Surfaces and authority + +List every filesystem, repository, application, account, network, and +integration surface this workflow may read or write. Discovery and an existing +session grant no authority. The setup scaffold writes filesystem targets only; +describe any later external action as a separate runtime request with its own +proposal, approval, capable tool, and result. + +## Approval + +Name the exact write targets and effects. State when existing user +authorization applies and which proposal changes require renewed approval. + +## Workflow + +Describe the smallest deterministic sequence that produces the approved +result. Keep every effect inside the approved boundary. + +## Idempotence, retries, and failure + +Treat matching output as a no-op. Stop on divergent existing content, path +escape, symbolic links, partial writes, or an unapproved surface. Do not +silently retry or overwrite. + +## Durable outputs and provenance + +Name the files or records that persist, the evidence they retain, and their +authority. Exclude credentials, tokens, cookies, session data, and unrelated +ambient context. + +## Verification + +Name the narrow checks that demonstrate the approved result and the KB +maintenance required after durable edits. diff --git a/src/authoring.test.ts b/src/authoring.test.ts index 70aaa15..a9c9a81 100644 --- a/src/authoring.test.ts +++ b/src/authoring.test.ts @@ -33,7 +33,11 @@ import { type AuthoringDependencies, type AuthoringOptions, } from "./authoring.js"; -import { analyzeVault, parseNote } from "./graph.js"; +import { + analyzeVault, + isCanonicalRelationPredicate, + parseNote, +} from "./graph.js"; const fixtures: string[] = []; @@ -966,5 +970,7 @@ describe("single-note authoring", () => { expect(listNoteRelations(root, "notes/source")) .rejects.toThrow("canonical kebab-case"); expect(normalizeRelationPredicate(" Depends_On ")).toBe("depends-on"); + expect(isCanonicalRelationPredicate(normalizeRelationPredicate("Evidence_By"))) + .toBe(true); }); }); diff --git a/src/authoring.ts b/src/authoring.ts index 832fe23..2028827 100644 --- a/src/authoring.ts +++ b/src/authoring.ts @@ -35,7 +35,10 @@ import { type NoteLock, type NoteLockOptions, } from "./note-lock.js"; -import { isCanonicalNoteId } from "./graph.js"; +import { + isCanonicalNoteId, + isCanonicalRelationPredicate, +} from "./graph.js"; import { parseDocumentId, parseQualifiedDocumentUri, @@ -43,7 +46,6 @@ import { const MAX_NOTE_BYTES = 16 * 1024 * 1024; const NOTE_REVISION_PATTERN = /^sha256:[0-9a-f]{64}$/u; -const PREDICATE_PATTERN = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u; const MAX_PARENT_DIRECTORY_ENTRIES = 100_000; const MAX_RECOVERY_LOCATIONS_PER_NOTE = 8; @@ -259,7 +261,7 @@ export function normalizeRelationPredicate(value: string): string { .replaceAll("_", "-") .replace(/\s+/gu, "-") .replace(/-{2,}/gu, "-"); - if (!PREDICATE_PATTERN.test(normalized)) { + if (!isCanonicalRelationPredicate(normalized)) { throw new TypeError(`not a valid relation predicate: ${JSON.stringify(value)}`); } return normalized; diff --git a/src/cli.test.ts b/src/cli.test.ts index a345ab8..cb6fc0a 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -16,6 +16,7 @@ import { MAX_QUERY_TEXT_UTF8_BYTES, } from "./query.js"; import type { KnowledgeBaseSession } from "./sdk.js"; +import { parsePercolationCliOutput } from "./percolate.js"; import { MAX_SEARCH_NOTE_REFERENCE_BYTES, MAX_SEARCH_RELATED_SEEDS, @@ -1394,6 +1395,8 @@ describe("kb vault commands", () => { "Alpha", "--tag", "agent-memory", + "--tag", + "shared-signal", "--body", "# Alpha\n\nA durable write path.\n", "--root", @@ -1440,6 +1443,8 @@ describe("kb vault commands", () => { "Gamma", "--tag", "agent-memory", + "--tag", + "shared-signal", "--root", vault, "--json", @@ -1539,13 +1544,37 @@ describe("kb vault commands", () => { expect(percolation).toMatchObject({ note: "notes/alpha", minSupport: 2, + limit: 25, + schemaVersion: 2, }); + expect(parsePercolationCliOutput(percolation)).toEqual(percolation); expect(arrayProperty(percolation, "candidates")).toContainEqual(expect.objectContaining({ kind: "missing-concept", tag: "agent-memory", suggestedId: "notes/agent-memory", support: 3, })); + expect(arrayProperty(percolation, "candidates")).toContainEqual(expect.objectContaining({ + kind: "missing-relation", + source: "notes/alpha", + target: "notes/gamma", + predicate: { kind: "required" }, + support: 2, + })); + + const terminalPercolation = captureOutput(); + expect(await main([ + "percolate", + "notes/alpha", + "--root", + vault, + "--min-support", + "2", + ], terminalPercolation.output)).toBe(0); + expect(terminalPercolation.stdout()).toContain( + "relation pair {notes/alpha, notes/gamma} (predicate required; 2 shared signals)", + ); + expect(terminalPercolation.stdout()).not.toContain("notes/alpha → notes/gamma"); const removeOutput = captureOutput(); expect(await main([ diff --git a/src/cli.ts b/src/cli.ts index 8c6a0ee..1e1300e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -81,6 +81,7 @@ import { MAX_PERCOLATION_NOTES, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, percolateVault, + type PercolationCliOutputV2, type PercolationResult, } from "./percolate.js"; import { @@ -2693,7 +2694,7 @@ function renderPercolation(result: PercolationResult, note: string | undefined): ); } else if (candidate.kind === "missing-relation") { lines.push( - ` relation ${safe(candidate.source)} ${safe(candidate.suggestedPredicate)} ${safe(candidate.target)} (${candidate.support} shared signals)`, + ` relation pair {${safe(candidate.source)}, ${safe(candidate.target)}} (predicate required; ${candidate.support} shared signals)`, ); } else if (candidate.kind === "unlinked-mention") { lines.push( @@ -2754,13 +2755,17 @@ async function runPercolate( limit: command.limit, }, ); + const jsonOutput: PercolationCliOutputV2 = { + root: snapshot.root, + note: command.note ?? null, + minSupport: command.minSupport, + limit: command.limit, + schemaVersion: result.schemaVersion, + candidates: result.candidates, + truncated: result.truncated, + }; output.stdout(command.json - ? terminalSafeJson({ - root: snapshot.root, - note: command.note ?? null, - minSupport: command.minSupport, - ...result, - }) + ? terminalSafeJson(jsonOutput) : sanitizeTerminalText(renderPercolation(result, command.note))); return 0; } diff --git a/src/graph.test.ts b/src/graph.test.ts index 1566e69..512a544 100644 --- a/src/graph.test.ts +++ b/src/graph.test.ts @@ -4,6 +4,7 @@ import { analyzeVault, catalogEnd, catalogStart, + isCanonicalRelationPredicate, lookupNote, metadataValueFromUnknown, parseNote, @@ -14,6 +15,15 @@ import { } from "./graph.js"; describe("note parsing", () => { + test("shares one exact canonical predicate language with authoring", () => { + expect(isCanonicalRelationPredicate("evidenced-by")).toBe(true); + expect(isCanonicalRelationPredicate("related-to")).toBe(true); + expect(isCanonicalRelationPredicate("custom-predicate-2")).toBe(true); + expect(isCanonicalRelationPredicate("Related-To")).toBe(false); + expect(isCanonicalRelationPredicate("related_to")).toBe(false); + expect(isCanonicalRelationPredicate("e\u0301vidence")).toBe(false); + }); + test("reads Obsidian properties and ignores links in code and comments", () => { const note = parseNote("notes/context.md", [ "---", diff --git a/src/graph.ts b/src/graph.ts index 91ff8c9..45531cc 100644 --- a/src/graph.ts +++ b/src/graph.ts @@ -327,6 +327,13 @@ type ParsedMetadata = { const relationPredicatePattern = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u; const maxNoteIdLength = 2_048; +/** Whether a value is the exact lower-kebab predicate accepted by the graph. */ +export function isCanonicalRelationPredicate(value: string): boolean { + return value !== "" + && value === value.normalize("NFC") + && relationPredicatePattern.test(value); +} + /** * Whether a value is the exact, extensionless vault-root ID used on disk. * @@ -476,7 +483,7 @@ function parsedRelations( } const predicate = pair.key.value.normalize("NFC"); - if (!relationPredicatePattern.test(predicate)) { + if (!isCanonicalRelationPredicate(predicate)) { relationIssues.push(malformedRelation( source, predicateLine, diff --git a/src/percolate.test.ts b/src/percolate.test.ts index 5988842..970de02 100644 --- a/src/percolate.test.ts +++ b/src/percolate.test.ts @@ -2,9 +2,17 @@ import { describe, expect, test } from "bun:test"; import { MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + PERCOLATION_RESULT_SCHEMA_VERSION, + parsePercolationCliOutput, + parsePercolationCliOutputV1, + parsePercolationResult, + parsePercolationResultV1, percolateVault, type MissingConceptCandidate, type MissingRelationCandidate, + type PercolationCliOutputV1, + type PercolationCliOutputV2, + type PercolationResultV1, } from "./percolate.js"; import { analyzeVault, @@ -99,9 +107,139 @@ describe("read-only graph percolation", () => { ); expect(shared).toBeDefined(); expect(shared?.support).toBe(4); + expect(shared?.predicate).toEqual({ kind: "required" }); expect(new Set(shared?.evidence.map((evidence) => evidence.kind))).toEqual( new Set(["shared-concept", "shared-tag"]), ); + expect(result.schemaVersion).toBe(PERCOLATION_RESULT_SCHEMA_VERSION); + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.candidates)).toBe(true); + expect(Object.isFrozen(shared)).toBe(true); + expect(Object.isFrozen(shared?.evidence)).toBe(true); + }); + + test("parses V2 and historical V1 as distinct immutable contracts", () => { + const v2 = percolateVault(discoveryFixture(), analyzeVault(discoveryFixture())); + expect(parsePercolationResult(v2)).toEqual(v2); + expect(() => parsePercolationResultV1(v2)).toThrow("exactly"); + + const v1: PercolationResultV1 = { + candidates: v2.candidates.map((candidate) => + candidate.kind === "missing-relation" + ? { + kind: candidate.kind, + source: candidate.source, + target: candidate.target, + suggestedPredicate: "related-to" as const, + support: candidate.support, + evidenceTruncated: candidate.evidenceTruncated, + evidence: candidate.evidence, + } + : candidate), + truncated: v2.truncated, + }; + const parsedV1 = parsePercolationResultV1(v1); + expect(parsedV1).toEqual(v1); + expect(Object.isFrozen(parsedV1)).toBe(true); + expect(Object.isFrozen(parsedV1.candidates)).toBe(true); + expect(() => parsePercolationResult(v1)).toThrow("exactly"); + + const cliV1: PercolationCliOutputV1 = { + root: "/vault", + note: "Alpha lookup", + minSupport: 2, + candidates: parsedV1.candidates, + truncated: parsedV1.truncated, + }; + const cliV2: PercolationCliOutputV2 = { + root: "/vault", + note: "Alpha lookup", + minSupport: 2, + limit: 25, + schemaVersion: 2, + candidates: v2.candidates, + truncated: v2.truncated, + }; + expect(parsePercolationCliOutputV1(cliV1)).toEqual(cliV1); + expect(parsePercolationCliOutput(cliV2)).toEqual(cliV2); + expect(() => parsePercolationCliOutputV1({ + ...cliV1, + minSupport: 1, + })).toThrow("from 2 through 1000"); + expect(() => parsePercolationCliOutput({ + ...cliV2, + minSupport: 1, + })).toThrow("from 2 through 1000"); + expect(() => parsePercolationCliOutput(cliV1)).toThrow("exactly"); + expect(() => parsePercolationCliOutputV1(cliV2)).toThrow("exactly"); + expect(() => parsePercolationResult(cliV2)).toThrow("exactly"); + }); + + test("rejects structural capabilities, ambiguity, and inconsistent evidence", () => { + const notes = discoveryFixture(); + const valid = percolateVault(notes, analyzeVault(notes)); + const relation = valid.candidates.find((candidate) => + candidate.kind === "missing-relation"); + expect(relation).toBeDefined(); + if (relation === undefined) throw new Error("missing fixture relation"); + + expect(parsePercolationResult({ + ...valid, + candidates: valid.candidates.map((candidate) => + candidate === relation + ? { ...candidate, predicate: { kind: "suggested", value: "custom-predicate-2" } } + : candidate), + }).candidates).toContainEqual(expect.objectContaining({ + kind: "missing-relation", + predicate: { kind: "suggested", value: "custom-predicate-2" }, + })); + + for (const [source, target] of [ + [relation.target, relation.source], + [relation.source, relation.source], + ] as const) { + expect(() => parsePercolationResult({ + ...valid, + candidates: valid.candidates.map((candidate) => + candidate === relation ? { ...candidate, source, target } : candidate), + })).toThrow("ordered, distinct pair"); + } + + expect(() => parsePercolationResult({ ...valid, extra: true })).toThrow("exactly"); + expect(() => parsePercolationResult(Object.assign( + Object.create({ inherited: true }) as object, + valid, + ))).toThrow("plain data object"); + const accessor = { ...valid } as Record; + Object.defineProperty(accessor, "truncated", { + enumerable: true, + get: () => false, + }); + expect(() => parsePercolationResult(accessor)).toThrow("data property"); + expect(() => parsePercolationResult({ + ...valid, + candidates: [ + ...valid.candidates, + ...Array.from({ length: 1_001 - valid.candidates.length }, () => relation), + ], + })).toThrow("1,000-entry limit"); + expect(() => parsePercolationResult({ + ...valid, + candidates: valid.candidates.map((candidate) => + candidate === relation + ? { + ...candidate, + evidence: candidate.evidence.map((evidence, index) => + index === 0 && "note" in evidence + ? { ...evidence, note: "notes/not-an-endpoint", path: "notes/not-an-endpoint.md" } + : evidence), + } + : candidate), + })).toThrow("unordered endpoints"); + expect(() => parsePercolationResult({ + ...valid, + candidates: valid.candidates.toReversed(), + })).toThrow("canonical percolation ordering"); }); test("turns graph mentions into sourced candidates and respects explicit edges", () => { @@ -144,7 +282,7 @@ describe("read-only graph percolation", () => { "relations:", " self-check: [notes/a]", " mirrors: [notes/b]", - " broken: [notes/missing]", + " broken: [notes/missing, notes/missing]", " Malformed: [notes/b]", "---", "# Alpha", @@ -166,6 +304,8 @@ describe("read-only graph percolation", () => { expect(problems).toContain("reciprocal-relation"); expect(problems).toContain("broken-relation"); expect(problems).toContain("malformed-relation"); + expect(problems.filter((problem) => problem === "broken-relation")) + .toHaveLength(1); expect(result.candidates .filter((candidate) => candidate.kind === "relation-hygiene") .every((candidate) => candidate.evidence.length > 0)).toBe(true); diff --git a/src/percolate.ts b/src/percolate.ts index af1f37a..41ac0f6 100644 --- a/src/percolate.ts +++ b/src/percolate.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { posix } from "node:path"; import { + isCanonicalNoteId, + isCanonicalRelationPredicate, lookupNote, MAX_ANALYZED_NOTES, MAX_MENTIONS, @@ -18,6 +20,10 @@ export const MAX_PERCOLATION_MENTION_PAIRS = 250_000; export const MAX_PERCOLATION_MENTIONS = MAX_MENTIONS; export const MAX_SCOPED_PERCOLATION_MENTION_PAIRS = MAX_PERCOLATION_NOTES * 2; +export const PERCOLATION_RESULT_SCHEMA_VERSION = 2 as const; +export const MAX_PERCOLATION_RESULT_NODES = 250_000; +export const MAX_PERCOLATION_RESULT_UTF8_BYTES = 16 * 1024 * 1024; +export const MAX_PERCOLATION_TEXT_UTF8_BYTES = 64 * 1024; const MAX_PERCOLATION_EVIDENCE = 250_000; const MAX_PERCOLATION_PAIR_OBSERVATIONS = MAX_PERCOLATION_MENTION_PAIRS; @@ -84,7 +90,12 @@ export type MissingConceptCandidate = { readonly evidence: readonly MissingConceptEvidence[]; }; -export type MissingRelationCandidate = { +export type PredicateDisposition = + | { readonly kind: "required" } + | { readonly kind: "suggested"; readonly value: string }; + +/** @deprecated Archival V1 shape; retained through the 0.18.x compatibility cycle. */ +export type MissingRelationCandidateV1 = { readonly kind: "missing-relation"; readonly source: string; readonly target: string; @@ -94,6 +105,20 @@ export type MissingRelationCandidate = { readonly evidence: readonly (SharedTagEvidence | SharedConceptEvidence)[]; }; +export type MissingRelationCandidateV2 = { + readonly kind: "missing-relation"; + /** Lexicographically ordered endpoint; this is not semantic direction. */ + readonly source: string; + /** Lexicographically ordered endpoint; this is not semantic direction. */ + readonly target: string; + readonly predicate: PredicateDisposition; + readonly support: number; + readonly evidenceTruncated: boolean; + readonly evidence: readonly (SharedTagEvidence | SharedConceptEvidence)[]; +}; + +export type MissingRelationCandidate = MissingRelationCandidateV2; + export type UnlinkedMentionCandidate = { readonly kind: "unlinked-mention"; readonly source: string; @@ -122,12 +147,20 @@ export type RelationHygieneCandidate = { readonly evidence: readonly (RelationEvidence | RelationIssueEvidence)[]; }; -export type PercolationCandidate = +export type PercolationCandidateV1 = + | MissingConceptCandidate + | MissingRelationCandidateV1 + | UnlinkedMentionCandidate + | RelationHygieneCandidate; + +export type PercolationCandidateV2 = | MissingConceptCandidate - | MissingRelationCandidate + | MissingRelationCandidateV2 | UnlinkedMentionCandidate | RelationHygieneCandidate; +export type PercolationCandidate = PercolationCandidateV2; + export type PercolateOptions = { /** Limit candidates to evidence involving this resolvable note. */ readonly note?: string; @@ -136,8 +169,43 @@ export type PercolateOptions = { readonly limit?: number; }; -export type PercolationResult = { - readonly candidates: readonly PercolationCandidate[]; +/** + * @deprecated Historical unversioned result retained for explicit archival + * parsing through 0.18.x; it may be removed no earlier than 0.19.0. + */ +export type PercolationResultV1 = { + readonly candidates: readonly PercolationCandidateV1[]; + readonly truncated: boolean; +}; + +export type PercolationResultV2 = { + readonly schemaVersion: typeof PERCOLATION_RESULT_SCHEMA_VERSION; + readonly candidates: readonly PercolationCandidateV2[]; + readonly truncated: boolean; +}; + +export type PercolationResult = PercolationResultV2; + +/** + * @deprecated Historical JSON envelope emitted by `kb percolate --json`, + * retained through 0.18.x and removable no earlier than 0.19.0. + */ +export type PercolationCliOutputV1 = { + readonly root: string; + readonly note: string | null; + readonly minSupport: number; + readonly candidates: readonly PercolationCandidateV1[]; + readonly truncated: boolean; +}; + +export type PercolationCliOutputV2 = { + readonly root: string; + /** The caller's free-form lookup text, not a canonical note identity. */ + readonly note: string | null; + readonly minSupport: number; + readonly limit: number; + readonly schemaVersion: typeof PERCOLATION_RESULT_SCHEMA_VERSION; + readonly candidates: readonly PercolationCandidateV2[]; readonly truncated: boolean; }; @@ -184,6 +252,7 @@ type AnalysisWithRelations = VaultAnalysis & { }; type SharedEvidence = SharedTagEvidence | SharedConceptEvidence; +type AnyPercolationCandidate = PercolationCandidateV1 | PercolationCandidateV2; type SharedAccumulation = { support: number; @@ -392,7 +461,7 @@ function compareSharedEvidence( || compareText(left.note, right.note); } -function candidateIdentity(candidate: PercolationCandidate): string { +function candidateIdentity(candidate: AnyPercolationCandidate): string { switch (candidate.kind) { case "missing-concept": return candidate.tag; @@ -406,11 +475,13 @@ function candidateIdentity(candidate: PercolationCandidate): string { candidate.source, candidate.predicate ?? "", candidate.target ?? "", + candidate.message, + String(candidate.evidence[0]?.line ?? 0), ].join("\u0000"); } } -const candidateKindRank: Readonly> = { +const candidateKindRank: Readonly> = { "relation-hygiene": 0, "unlinked-mention": 1, "missing-relation": 2, @@ -418,8 +489,8 @@ const candidateKindRank: Readonly> }; function compareCandidates( - left: PercolationCandidate, - right: PercolationCandidate, + left: AnyPercolationCandidate, + right: AnyPercolationCandidate, ): number { return right.support - left.support || candidateKindRank[left.kind] - candidateKindRank[right.kind] @@ -728,7 +799,7 @@ export function percolateVault( kind: "missing-relation", source, target, - suggestedPredicate: "related-to", + predicate: { kind: "required" }, support: accumulated.support, evidenceTruncated: accumulated.evidenceCount > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, @@ -867,11 +938,943 @@ export function percolateVault( `Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} candidate limit.`, ); } - const sorted = candidates + const uniqueCandidates = new Map(); + for (const candidate of candidates) { + const identity = `${candidate.kind}\u0000${candidateIdentity(candidate)}`; + if (!uniqueCandidates.has(identity)) uniqueCandidates.set(identity, candidate); + } + const sorted = [...uniqueCandidates.values()] .filter((candidate) => candidateInvolvesNote(candidate, noteFilter)) .toSorted(compareCandidates); - return { + return parsePercolationResultV2({ + schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION, candidates: sorted.slice(0, limit), truncated: sorted.length > limit, + }); +} + +type ParseBudget = { + nodes: number; + utf8Bytes: number; +}; + +type DataRecord = Readonly>; + +function countParseNode(budget: ParseBudget, label: string): void { + budget.nodes += 1; + if (budget.nodes > MAX_PERCOLATION_RESULT_NODES) { + throw new RangeError( + `${label} exceeds the ${MAX_PERCOLATION_RESULT_NODES.toLocaleString("en-US")}-node percolation result limit.`, + ); + } +} + +function dataRecord( + value: unknown, + label: string, + budget: ParseBudget, +): DataRecord { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`${label} must be a plain data object.`); + } + const prototype: unknown = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${label} must be a plain data object.`); + } + countParseNode(budget, label); + const output = Object.create(null) as Record; + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const key of Reflect.ownKeys(descriptors)) { + if (typeof key !== "string") { + throw new TypeError(`${label} must not contain symbol fields.`); + } + const descriptor = descriptors[key]; + if ( + descriptor === undefined + || !("value" in descriptor) + || !descriptor.enumerable + ) { + throw new TypeError(`${label}.${key} must be an enumerable data property.`); + } + Object.defineProperty(output, key, { + configurable: false, + enumerable: true, + value: descriptor.value, + writable: false, + }); + } + return Object.freeze(output); +} + +function exactKeys( + record: DataRecord, + keys: readonly string[], + label: string, +): void { + const actual = Reflect.ownKeys(record); + const expected = new Set(keys); + if ( + actual.length !== keys.length + || actual.some((key) => typeof key !== "string" || !expected.has(key)) + ) { + throw new TypeError(`${label} must contain exactly: ${keys.join(", ")}.`); + } +} + +function dataArray( + value: unknown, + label: string, + maximum: number, + budget: ParseBudget, +): readonly unknown[] { + if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) { + throw new TypeError(`${label} must be an ordinary array.`); + } + if (value.length > maximum) { + throw new RangeError( + `${label} exceeds its ${maximum.toLocaleString("en-US")}-entry limit.`, + ); + } + countParseNode(budget, label); + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const key of Reflect.ownKeys(descriptors)) { + if (typeof key !== "string") { + throw new TypeError(`${label} must not contain symbol fields.`); + } + if (key === "length") continue; + const index = Number(key); + if ( + !Number.isSafeInteger(index) + || index < 0 + || index >= value.length + || String(index) !== key + ) { + throw new TypeError(`${label} contains a non-index property.`); + } + } + const output: unknown[] = []; + for (let index = 0; index < value.length; index += 1) { + const descriptor = descriptors[String(index)]; + if ( + descriptor === undefined + || !("value" in descriptor) + || !descriptor.enumerable + ) { + throw new TypeError(`${label} must be a dense array of data properties.`); + } + output.push(descriptor.value); + } + return Object.freeze(output); +} + +function hasUnpairedSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (next < 0xdc00 || next > 0xdfff) return true; + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +} + +function parsedText( + value: unknown, + label: string, + budget: ParseBudget, + options: { readonly empty?: boolean } = {}, +): string { + if ( + typeof value !== "string" + || (options.empty !== true && value === "") + || hasUnpairedSurrogate(value) + ) { + throw new TypeError(`${label} must be a bounded Unicode string.`); + } + const bytes = new TextEncoder().encode(value).byteLength; + if (bytes > MAX_PERCOLATION_TEXT_UTF8_BYTES) { + throw new RangeError( + `${label} exceeds its ${MAX_PERCOLATION_TEXT_UTF8_BYTES.toLocaleString("en-US")}-byte limit.`, + ); + } + budget.utf8Bytes += bytes; + if (budget.utf8Bytes > MAX_PERCOLATION_RESULT_UTF8_BYTES) { + throw new RangeError( + `Percolation result text exceeds ${MAX_PERCOLATION_RESULT_UTF8_BYTES.toLocaleString("en-US")} UTF-8 bytes.`, + ); + } + return value; +} + +function canonicalNote( + value: unknown, + label: string, + budget: ParseBudget, +): string { + const parsed = parsedText(value, label, budget); + if (!isCanonicalNoteId(parsed)) { + throw new TypeError(`${label} must be a canonical note ID.`); + } + return parsed; +} + +function canonicalMarkdownPath( + value: unknown, + label: string, + budget: ParseBudget, +): string { + const parsed = parsedText(value, label, budget); + if ( + !parsed.endsWith(".md") + || !isCanonicalNoteId(parsed.slice(0, -3)) + ) { + throw new TypeError(`${label} must be a canonical vault Markdown path.`); + } + return parsed; +} + +function canonicalPredicate( + value: unknown, + label: string, + budget: ParseBudget, +): string { + const parsed = parsedText(value, label, budget); + if (!isCanonicalRelationPredicate(parsed)) { + throw new TypeError(`${label} must be a canonical relation predicate.`); + } + return parsed; +} + +function nullableText( + value: unknown, + label: string, + budget: ParseBudget, +): string | null { + return value === null ? null : parsedText(value, label, budget, { empty: true }); +} + +function parsedBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") throw new TypeError(`${label} must be a boolean.`); + return value; +} + +function positiveSafeInteger( + value: unknown, + label: string, + maximum = MAX_PERCOLATION_EVIDENCE, +): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < 1 + || value > maximum + ) { + throw new TypeError(`${label} must be a positive bounded safe integer.`); + } + return value; +} + +function parsedMinSupport(value: unknown, label: string): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < DEFAULT_PERCOLATION_MIN_SUPPORT + || value > MAX_PERCOLATION_LIMIT + ) { + throw new TypeError( + `${label} must be an integer from ${DEFAULT_PERCOLATION_MIN_SUPPORT} through ${MAX_PERCOLATION_LIMIT}.`, + ); + } + return value; +} + +function predicateDisposition( + value: unknown, + label: string, + budget: ParseBudget, +): PredicateDisposition { + const record = dataRecord(value, label, budget); + const kind = parsedText(record.kind, `${label}.kind`, budget); + if (kind === "required") { + exactKeys(record, ["kind"], label); + return Object.freeze({ kind: "required" }); + } + if (kind === "suggested") { + exactKeys(record, ["kind", "value"], label); + return Object.freeze({ + kind: "suggested", + value: canonicalPredicate(record.value, `${label}.value`, budget), + }); + } + throw new TypeError(`${label}.kind must be required or suggested.`); +} + +function parsedMissingConceptEvidence( + value: unknown, + label: string, + budget: ParseBudget, +): MissingConceptEvidence { + const record = dataRecord(value, label, budget); + exactKeys(record, ["kind", "note", "path", "tag"], label); + if (parsedText(record.kind, `${label}.kind`, budget) !== "tag") { + throw new TypeError(`${label}.kind must be tag.`); + } + const note = canonicalNote(record.note, `${label}.note`, budget); + const path = canonicalMarkdownPath(record.path, `${label}.path`, budget); + if (path !== `${note}.md`) throw new TypeError(`${label}.path must identify its note.`); + return Object.freeze({ + kind: "tag", + note, + path, + tag: parsedText(record.tag, `${label}.tag`, budget), + }); +} + +function parsedSharedEvidence( + value: unknown, + label: string, + budget: ParseBudget, +): SharedEvidence { + const record = dataRecord(value, label, budget); + const kind = parsedText(record.kind, `${label}.kind`, budget); + if (kind === "shared-tag") { + exactKeys(record, ["kind", "note", "path", "tag"], label); + const note = canonicalNote(record.note, `${label}.note`, budget); + const path = canonicalMarkdownPath(record.path, `${label}.path`, budget); + if (path !== `${note}.md`) throw new TypeError(`${label}.path must identify its note.`); + return Object.freeze({ + kind: "shared-tag", + note, + path, + tag: parsedText(record.tag, `${label}.tag`, budget), + }); + } + if (kind === "shared-concept") { + exactKeys( + record, + ["kind", "note", "path", "concept", "conceptPath"], + label, + ); + const note = canonicalNote(record.note, `${label}.note`, budget); + const path = canonicalMarkdownPath(record.path, `${label}.path`, budget); + const concept = canonicalNote(record.concept, `${label}.concept`, budget); + const conceptPath = canonicalMarkdownPath( + record.conceptPath, + `${label}.conceptPath`, + budget, + ); + if (path !== `${note}.md`) throw new TypeError(`${label}.path must identify its note.`); + if (conceptPath !== `${concept}.md`) { + throw new TypeError(`${label}.conceptPath must identify its concept.`); + } + return Object.freeze({ + kind: "shared-concept", + note, + path, + concept, + conceptPath, + }); + } + throw new TypeError(`${label}.kind must be shared-tag or shared-concept.`); +} + +function parsedMentionEvidence( + value: unknown, + label: string, + budget: ParseBudget, +): MentionEvidence { + const record = dataRecord(value, label, budget); + exactKeys(record, ["kind", "source", "target", "line", "phrase"], label); + if (parsedText(record.kind, `${label}.kind`, budget) !== "mention") { + throw new TypeError(`${label}.kind must be mention.`); + } + return Object.freeze({ + kind: "mention", + source: canonicalNote(record.source, `${label}.source`, budget), + target: canonicalNote(record.target, `${label}.target`, budget), + line: positiveSafeInteger(record.line, `${label}.line`), + phrase: parsedText(record.phrase, `${label}.phrase`, budget), + }); +} + +function parsedRelationEvidence( + value: unknown, + label: string, + budget: ParseBudget, +): RelationEvidence { + const record = dataRecord(value, label, budget); + exactKeys( + record, + ["kind", "source", "target", "predicate", "line", "authoredTarget"], + label, + ); + if (parsedText(record.kind, `${label}.kind`, budget) !== "relation") { + throw new TypeError(`${label}.kind must be relation.`); + } + return Object.freeze({ + kind: "relation", + source: canonicalNote(record.source, `${label}.source`, budget), + target: canonicalNote(record.target, `${label}.target`, budget), + predicate: canonicalPredicate(record.predicate, `${label}.predicate`, budget), + line: positiveSafeInteger(record.line, `${label}.line`), + authoredTarget: parsedText(record.authoredTarget, `${label}.authoredTarget`, budget), + }); +} + +function parsedRelationIssueEvidence( + value: unknown, + label: string, + budget: ParseBudget, +): RelationIssueEvidence { + const record = dataRecord(value, label, budget); + exactKeys( + record, + [ + "kind", + "issue", + "source", + "line", + "predicate", + "target", + "candidates", + "candidatesTruncated", + "message", + ], + label, + ); + if (parsedText(record.kind, `${label}.kind`, budget) !== "relation-issue") { + throw new TypeError(`${label}.kind must be relation-issue.`); + } + if ( + record.issue !== "malformed" + && record.issue !== "broken" + && record.issue !== "ambiguous" + ) { + throw new TypeError(`${label}.issue is unsupported.`); + } + const issue = parsedText(record.issue, `${label}.issue`, budget) as + RelationIssueEvidence["issue"]; + const candidates = dataArray( + record.candidates, + `${label}.candidates`, + MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + budget, + ).map((candidate, index) => + canonicalNote(candidate, `${label}.candidates[${index}]`, budget)); + for (let index = 0; index < candidates.length; index += 1) { + const previous = candidates[index - 1]; + const candidate = candidates[index]; + if (candidate === undefined) continue; + if (previous !== undefined && compareText(previous, candidate) >= 0) { + throw new TypeError(`${label}.candidates must be sorted and unique.`); + } + } + if (issue !== "ambiguous" && candidates.length !== 0) { + throw new TypeError(`${label}.candidates are only valid for ambiguous issues.`); + } + if (issue === "ambiguous" && candidates.length < 2) { + throw new TypeError(`${label}.candidates must identify at least two ambiguous notes.`); + } + const candidatesTruncated = parsedBoolean( + record.candidatesTruncated, + `${label}.candidatesTruncated`, + ); + if ( + (issue !== "ambiguous" && candidatesTruncated) + || (candidatesTruncated + && candidates.length !== MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE) + ) { + throw new TypeError(`${label}.candidatesTruncated is inconsistent.`); + } + const predicate = issue === "malformed" + ? nullableText(record.predicate, `${label}.predicate`, budget) + : canonicalPredicate(record.predicate, `${label}.predicate`, budget); + const target = issue === "malformed" + ? nullableText(record.target, `${label}.target`, budget) + : canonicalNote(record.target, `${label}.target`, budget); + return Object.freeze({ + kind: "relation-issue", + issue, + source: canonicalNote(record.source, `${label}.source`, budget), + line: positiveSafeInteger(record.line, `${label}.line`), + predicate, + target, + candidates: Object.freeze(candidates), + candidatesTruncated, + message: parsedText(record.message, `${label}.message`, budget), + }); +} + +function evidenceArray( + value: unknown, + label: string, + budget: ParseBudget, + parse: (entry: unknown, label: string, budget: ParseBudget) => T, +): readonly T[] { + const input = dataArray( + value, + label, + MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + budget, + ); + if (input.length === 0) throw new TypeError(`${label} must not be empty.`); + const output = input.map((entry, index) => + parse(entry, `${label}[${index}]`, budget)); + const identities = new Set(); + for (const entry of output) { + const identity = JSON.stringify(entry); + if (identities.has(identity)) throw new TypeError(`${label} must be unique.`); + identities.add(identity); + } + return Object.freeze(output); +} + +function parsedRelationProblem( + value: unknown, + label: string, + budget: ParseBudget, +): RelationHygieneProblem { + const parsed = parsedText(value, label, budget); + if ( + parsed !== "self-relation" + && parsed !== "reciprocal-relation" + && parsed !== "malformed-relation" + && parsed !== "broken-relation" + && parsed !== "ambiguous-relation" + ) throw new TypeError(`${label} is unsupported.`); + return parsed; +} + +function parsedCommonCandidate( + record: DataRecord, + label: string, +): { + readonly support: number; + readonly evidenceTruncated: boolean; +} { + return { + support: positiveSafeInteger(record.support, `${label}.support`), + evidenceTruncated: parsedBoolean( + record.evidenceTruncated, + `${label}.evidenceTruncated`, + ), + }; +} + +function parseCandidate( + value: unknown, + label: string, + budget: ParseBudget, + version: 1 | 2, +): PercolationCandidateV1 | PercolationCandidateV2 { + const record = dataRecord(value, label, budget); + const kind = parsedText(record.kind, `${label}.kind`, budget); + if (kind === "missing-concept") { + exactKeys( + record, + [ + "kind", + "tag", + "suggestedId", + "collidesWith", + "support", + "evidenceTruncated", + "evidence", + ], + label, + ); + const common = parsedCommonCandidate(record, label); + const tag = parsedText(record.tag, `${label}.tag`, budget); + const evidence = evidenceArray( + record.evidence, + `${label}.evidence`, + budget, + parsedMissingConceptEvidence, + ); + if (evidence.some((entry) => entry.tag !== tag)) { + throw new TypeError(`${label}.evidence must support the candidate tag.`); + } + if ( + (!common.evidenceTruncated && common.support !== evidence.length) + || (common.evidenceTruncated && common.support <= evidence.length) + ) { + throw new TypeError(`${label}.support does not match its bounded evidence.`); + } + return Object.freeze({ + kind: "missing-concept", + tag, + suggestedId: canonicalNote(record.suggestedId, `${label}.suggestedId`, budget), + collidesWith: record.collidesWith === null + ? null + : canonicalNote(record.collidesWith, `${label}.collidesWith`, budget), + ...common, + evidence, + }); + } + if (kind === "missing-relation") { + exactKeys( + record, + version === 1 + ? [ + "kind", + "source", + "target", + "suggestedPredicate", + "support", + "evidenceTruncated", + "evidence", + ] + : [ + "kind", + "source", + "target", + "predicate", + "support", + "evidenceTruncated", + "evidence", + ], + label, + ); + const source = canonicalNote(record.source, `${label}.source`, budget); + const target = canonicalNote(record.target, `${label}.target`, budget); + if (compareText(source, target) >= 0) { + throw new TypeError(`${label} endpoints must be an ordered, distinct pair.`); + } + const common = parsedCommonCandidate(record, label); + const evidence = evidenceArray( + record.evidence, + `${label}.evidence`, + budget, + parsedSharedEvidence, + ); + if (evidence.some((entry) => entry.note !== source && entry.note !== target)) { + throw new TypeError(`${label}.evidence must belong to one of the unordered endpoints.`); + } + const signalEndpoints = new Map>(); + for (const entry of evidence) { + const signal = entry.kind === "shared-tag" + ? `tag\u0000${entry.tag}` + : `concept\u0000${entry.concept}`; + const endpoints = signalEndpoints.get(signal) ?? new Set(); + endpoints.add(entry.note); + signalEndpoints.set(signal, endpoints); + } + if ([...signalEndpoints.values()].some((endpoints) => + endpoints.size !== 2 || !endpoints.has(source) || !endpoints.has(target))) { + throw new TypeError(`${label}.evidence must pair both unordered endpoints per signal.`); + } + if ( + (!common.evidenceTruncated && common.support !== signalEndpoints.size) + || (common.evidenceTruncated + && ( + evidence.length !== MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE + || common.support <= signalEndpoints.size + )) + ) { + throw new TypeError(`${label}.support does not match its bounded shared signals.`); + } + if (version === 1) { + if ( + parsedText( + record.suggestedPredicate, + `${label}.suggestedPredicate`, + budget, + ) !== "related-to" + ) { + throw new TypeError(`${label}.suggestedPredicate must be related-to.`); + } + return Object.freeze({ + kind: "missing-relation", + source, + target, + suggestedPredicate: "related-to", + ...common, + evidence, + }); + } + return Object.freeze({ + kind: "missing-relation", + source, + target, + predicate: predicateDisposition(record.predicate, `${label}.predicate`, budget), + ...common, + evidence, + }); + } + if (kind === "unlinked-mention") { + exactKeys( + record, + ["kind", "source", "target", "support", "evidenceTruncated", "evidence"], + label, + ); + const source = canonicalNote(record.source, `${label}.source`, budget); + const target = canonicalNote(record.target, `${label}.target`, budget); + const common = parsedCommonCandidate(record, label); + const evidence = evidenceArray( + record.evidence, + `${label}.evidence`, + budget, + parsedMentionEvidence, + ); + if (evidence.some((entry) => entry.source !== source || entry.target !== target)) { + throw new TypeError(`${label}.evidence must identify the candidate endpoints.`); + } + if ( + (!common.evidenceTruncated && common.support !== evidence.length) + || (common.evidenceTruncated && common.support <= evidence.length) + ) { + throw new TypeError(`${label}.support does not match its bounded evidence.`); + } + return Object.freeze({ + kind: "unlinked-mention", + source, + target, + ...common, + evidence, + }); + } + if (kind === "relation-hygiene") { + exactKeys( + record, + [ + "kind", + "problem", + "source", + "target", + "predicate", + "message", + "support", + "evidenceTruncated", + "evidence", + ], + label, + ); + const problem = parsedRelationProblem(record.problem, `${label}.problem`, budget); + const source = canonicalNote(record.source, `${label}.source`, budget); + const target = problem === "malformed-relation" + ? nullableText(record.target, `${label}.target`, budget) + : record.target === null + ? null + : canonicalNote(record.target, `${label}.target`, budget); + const predicate = problem === "malformed-relation" + ? nullableText(record.predicate, `${label}.predicate`, budget) + : record.predicate === null + ? null + : canonicalPredicate(record.predicate, `${label}.predicate`, budget); + const common = parsedCommonCandidate(record, label); + const relationProblem = problem === "self-relation" + || problem === "reciprocal-relation"; + const evidence = relationProblem + ? evidenceArray( + record.evidence, + `${label}.evidence`, + budget, + parsedRelationEvidence, + ) + : evidenceArray( + record.evidence, + `${label}.evidence`, + budget, + parsedRelationIssueEvidence, + ); + if (common.support !== evidence.length) { + throw new TypeError(`${label}.support must equal its hygiene evidence count.`); + } + if (relationProblem) { + const relations = evidence as readonly RelationEvidence[]; + if ( + target === null + || predicate === null + || common.evidenceTruncated + || (problem === "self-relation" && target !== source) + || (problem === "reciprocal-relation" + && (compareText(source, target) >= 0 || relations.length !== 2)) + || relations.some((entry) => + entry.predicate !== predicate + || (problem === "self-relation" + ? entry.source !== source || entry.target !== target + : !( + (entry.source === source && entry.target === target) + || (entry.source === target && entry.target === source) + ))) + ) throw new TypeError(`${label}.evidence must identify the hygiene relation.`); + } else { + const issues = evidence as readonly RelationIssueEvidence[]; + const expectedIssue = problem.slice(0, -"-relation".length); + if (issues.some((entry) => + entry.source !== source + || entry.issue !== expectedIssue + || entry.predicate !== predicate + || entry.target !== target + || entry.message !== record.message) + || common.evidenceTruncated + !== issues.some((entry) => entry.candidatesTruncated)) { + throw new TypeError(`${label}.evidence must identify the hygiene issue.`); + } + } + return Object.freeze({ + kind: "relation-hygiene", + problem, + source, + target, + predicate, + message: parsedText(record.message, `${label}.message`, budget), + ...common, + evidence, + }); + } + throw new TypeError(`${label}.kind is unsupported.`); +} + +function parsedCandidates( + value: unknown, + label: string, + budget: ParseBudget, + version: V, +): readonly (V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2)[] { + const input = dataArray(value, label, MAX_PERCOLATION_LIMIT, budget); + const output = input.map((entry, index) => + parseCandidate(entry, `${label}[${index}]`, budget, version)); + const identities = new Set(); + for (let index = 0; index < output.length; index += 1) { + const candidate = output[index]; + if (candidate === undefined) continue; + const identity = `${candidate.kind}\u0000${candidateIdentity(candidate)}`; + if (identities.has(identity)) throw new TypeError(`${label} must be unique.`); + identities.add(identity); + const previous = output[index - 1]; + if (previous !== undefined && compareCandidates(previous, candidate) > 0) { + throw new TypeError(`${label} must use canonical percolation ordering.`); + } + } + return Object.freeze(output) as readonly ( + V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2 + )[]; +} + +function parseResultFields( + record: DataRecord, + label: string, + budget: ParseBudget, + version: V, +): { + readonly candidates: readonly ( + V extends 1 ? PercolationCandidateV1 : PercolationCandidateV2 + )[]; + readonly truncated: boolean; +} { + return { + candidates: parsedCandidates(record.candidates, `${label}.candidates`, budget, version), + truncated: parsedBoolean(record.truncated, `${label}.truncated`), }; } + +/** + * Parse only the exact historical unversioned core result shape. This parser + * deliberately preserves `related-to`; it never guesses a V2 disposition. + * @deprecated Retained through 0.18.x; removable no earlier than 0.19.0. + */ +export function parsePercolationResultV1(value: unknown): PercolationResultV1 { + const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 }; + const record = dataRecord(value, "percolation result v1", budget); + exactKeys(record, ["candidates", "truncated"], "percolation result v1"); + const fields = parseResultFields(record, "percolation result v1", budget, 1); + return Object.freeze({ + candidates: fields.candidates, + truncated: fields.truncated, + }); +} + +/** Parse only the exact V2 core result shape; CLI envelopes are rejected. */ +export function parsePercolationResultV2(value: unknown): PercolationResultV2 { + const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 }; + const record = dataRecord(value, "percolation result v2", budget); + exactKeys( + record, + ["schemaVersion", "candidates", "truncated"], + "percolation result v2", + ); + if (record.schemaVersion !== PERCOLATION_RESULT_SCHEMA_VERSION) { + throw new TypeError("percolation result v2.schemaVersion must be 2."); + } + const fields = parseResultFields(record, "percolation result v2", budget, 2); + return Object.freeze({ + schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION, + candidates: fields.candidates, + truncated: fields.truncated, + }); +} + +export const parsePercolationResult = parsePercolationResultV2; + +/** + * Parse only the exact historical V1 CLI envelope without upgrading it. + * @deprecated Retained through 0.18.x; removable no earlier than 0.19.0. + */ +export function parsePercolationCliOutputV1(value: unknown): PercolationCliOutputV1 { + const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 }; + const label = "percolation CLI output v1"; + const record = dataRecord(value, label, budget); + exactKeys( + record, + ["root", "note", "minSupport", "candidates", "truncated"], + label, + ); + const fields = parseResultFields(record, label, budget, 1); + return Object.freeze({ + root: parsedText(record.root, `${label}.root`, budget), + note: nullableText(record.note, `${label}.note`, budget), + minSupport: parsedMinSupport(record.minSupport, `${label}.minSupport`), + candidates: fields.candidates, + truncated: fields.truncated, + }); +} + +/** Parse only the exact current V2 CLI envelope. */ +export function parsePercolationCliOutputV2(value: unknown): PercolationCliOutputV2 { + const budget: ParseBudget = { nodes: 0, utf8Bytes: 0 }; + const label = "percolation CLI output v2"; + const record = dataRecord(value, label, budget); + exactKeys( + record, + [ + "root", + "note", + "minSupport", + "limit", + "schemaVersion", + "candidates", + "truncated", + ], + label, + ); + if (record.schemaVersion !== PERCOLATION_RESULT_SCHEMA_VERSION) { + throw new TypeError(`${label}.schemaVersion must be 2.`); + } + const fields = parseResultFields(record, label, budget, 2); + const limit = positiveSafeInteger( + record.limit, + `${label}.limit`, + MAX_PERCOLATION_LIMIT, + ); + if ( + fields.candidates.length > limit + || (fields.truncated && fields.candidates.length !== limit) + ) { + throw new TypeError(`${label}.limit is inconsistent with its candidates.`); + } + return Object.freeze({ + root: parsedText(record.root, `${label}.root`, budget), + note: nullableText(record.note, `${label}.note`, budget), + minSupport: parsedMinSupport(record.minSupport, `${label}.minSupport`), + limit, + schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION, + candidates: fields.candidates, + truncated: fields.truncated, + }); +} + +export const parsePercolationCliOutput = parsePercolationCliOutputV2; From 4e38d9d3d121366f2702dd267b40e2825068515a Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sun, 30 Aug 2026 03:01:11 -0400 Subject: [PATCH 2/3] test: compare serialized percolation output --- src/cli.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli.test.ts b/src/cli.test.ts index cb6fc0a..5357ac0 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -1547,7 +1547,8 @@ describe("kb vault commands", () => { limit: 25, schemaVersion: 2, }); - expect(parsePercolationCliOutput(percolation)).toEqual(percolation); + expect(JSON.stringify(parsePercolationCliOutput(percolation))) + .toBe(JSON.stringify(percolation)); expect(arrayProperty(percolation, "candidates")).toContainEqual(expect.objectContaining({ kind: "missing-concept", tag: "agent-memory", From 9c00d434cde2a9a62d17a3c0d5999b513a43e484 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sun, 30 Aug 2026 03:38:21 -0400 Subject: [PATCH 3/3] fix: validate KB lifecycle release package --- dist/authoring.js | 4 +- dist/benchmark.js | 6 +- dist/cli.js | 28 +- dist/evaluation-builder.js | 10 +- dist/evaluation-kb.js | 10 +- dist/graph.js | 4 +- dist/{index-zxdy5pby.js => index-5m2ydj5q.js} | 4 +- dist/index-dyqwejk5.js | 531 -------- dist/{index-cxfrakt7.js => index-ekpwvbra.js} | 7 +- dist/{index-jsmvyyvf.js => index-ey46z1zf.js} | 8 +- dist/index-f9fy4w1n.js | 1121 +++++++++++++++++ dist/{index-cv6fh7z5.js => index-gm9t95d9.js} | 2 +- dist/{index-01jj6rbv.js => index-gxr0fctd.js} | 6 +- dist/{index-s2gw5aw9.js => index-qwgsmtsz.js} | 2 +- dist/{index-zzhgcwyt.js => index-vxmf14m1.js} | 6 +- dist/{index-n5dd7r0v.js => index-xw9ac71d.js} | 4 +- dist/{index-1vrd1rmn.js => index-ykvvkd77.js} | 2 +- dist/index.js | 38 +- dist/percolate.js | 24 +- dist/portfolio.js | 10 +- dist/sdk.js | 8 +- dist/search.js | 4 +- dist/semantic.js | 6 +- dist/workflows/decision-context.js | 10 +- dist/workflows/index.js | 10 +- scripts/kb-skill-contract.test.ts | 21 +- scripts/npm-package-identity.ts | 22 + scripts/npm-release-workflow.test.ts | 35 +- scripts/package-smoke.ts | 17 +- 29 files changed, 1340 insertions(+), 620 deletions(-) rename dist/{index-zxdy5pby.js => index-5m2ydj5q.js} (99%) delete mode 100644 dist/index-dyqwejk5.js rename dist/{index-cxfrakt7.js => index-ekpwvbra.js} (99%) rename dist/{index-jsmvyyvf.js => index-ey46z1zf.js} (99%) create mode 100644 dist/index-f9fy4w1n.js rename dist/{index-cv6fh7z5.js => index-gm9t95d9.js} (99%) rename dist/{index-01jj6rbv.js => index-gxr0fctd.js} (99%) rename dist/{index-s2gw5aw9.js => index-qwgsmtsz.js} (99%) rename dist/{index-zzhgcwyt.js => index-vxmf14m1.js} (99%) rename dist/{index-n5dd7r0v.js => index-xw9ac71d.js} (99%) rename dist/{index-1vrd1rmn.js => index-ykvvkd77.js} (97%) diff --git a/dist/authoring.js b/dist/authoring.js index 0268e47..0bc76b5 100644 --- a/dist/authoring.js +++ b/dist/authoring.js @@ -13,9 +13,9 @@ import { normalizeRelationPredicate, noteRevision, removeNoteRelation -} from "./index-01jj6rbv.js"; +} from "./index-gxr0fctd.js"; import"./index-3rm7cz6h.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; export { removeNoteRelation, noteRevision, diff --git a/dist/benchmark.js b/dist/benchmark.js index 8ad2442..69a6b15 100644 --- a/dist/benchmark.js +++ b/dist/benchmark.js @@ -4,13 +4,13 @@ import { createSyntheticRankFusionFixture, evaluateRanking, evaluateRetrievalBenchmark -} from "./index-s2gw5aw9.js"; -import"./index-cv6fh7z5.js"; +} from "./index-qwgsmtsz.js"; +import"./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; import"./index-5vwpzb5a.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; export { evaluateRetrievalBenchmark, evaluateRanking, diff --git a/dist/cli.js b/dist/cli.js index db19b3e..f91d021 100755 --- a/dist/cli.js +++ b/dist/cli.js @@ -13,7 +13,7 @@ import { loadPortfolioRegistry, openKnowledgePortfolio, snapshotPortfolioRegistry -} from "./index-jsmvyyvf.js"; +} from "./index-ey46z1zf.js"; import { diffCaptureBundle } from "./index-j4zgmzjr.js"; @@ -34,11 +34,11 @@ import { MAX_PERCOLATION_NOTES, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, percolateVault -} from "./index-dyqwejk5.js"; +} from "./index-f9fy4w1n.js"; import { knowledgeBaseEvaluationRetrieverIds, openKnowledgeBaseEvaluation -} from "./index-n5dd7r0v.js"; +} from "./index-xw9ac71d.js"; import { DEFAULT_SEARCH_RESULTS, MAX_SEARCH_CANDIDATES, @@ -46,7 +46,7 @@ import { MAX_SEARCH_RELATED_SEEDS, MAX_SEARCH_RESULTS, openKnowledgeBase -} from "./index-zzhgcwyt.js"; +} from "./index-vxmf14m1.js"; import { MAX_SEARCH_RULE_CONFIG_BYTES, parseSearchRules @@ -59,7 +59,7 @@ import { refreshVault, scanVault, sha256EmbeddingModelFile -} from "./index-zxdy5pby.js"; +} from "./index-5m2ydj5q.js"; import"./index-4j3tt0c3.js"; import"./index-1gwbassd.js"; import { @@ -79,11 +79,11 @@ import { addNoteRelation, createNote, removeNoteRelation -} from "./index-01jj6rbv.js"; +} from "./index-gxr0fctd.js"; import"./index-3rm7cz6h.js"; import { validateSearchQuery -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; import { navigateLinks } from "./index-d13v9ckt.js"; @@ -111,7 +111,7 @@ import { lookupNote, parseVaultKey, renderCatalog -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; import { main } from "./index-0kavxzqj.js"; @@ -2885,7 +2885,7 @@ function renderPercolation(result, note) { if (candidate.kind === "missing-concept") { lines.push(` concept #${safe(candidate.tag)} \u2192 ${safe(candidate.suggestedId)} (${candidate.support} supporting notes)` + (candidate.collidesWith === null ? "" : `; natural ID is occupied by ${safe(candidate.collidesWith)}`)); } else if (candidate.kind === "missing-relation") { - lines.push(` relation ${safe(candidate.source)} ${safe(candidate.suggestedPredicate)} ${safe(candidate.target)} (${candidate.support} shared signals)`); + lines.push(` relation pair {${safe(candidate.source)}, ${safe(candidate.target)}} (predicate required; ${candidate.support} shared signals)`); } else if (candidate.kind === "unlinked-mention") { lines.push(` mention ${safe(candidate.source)} \u2192 ${safe(candidate.target)} (${candidate.support})`); } else { @@ -2927,12 +2927,16 @@ async function runPercolate(command, output, dependencies) { minSupport: command.minSupport, limit: command.limit }); - output.stdout(command.json ? terminalSafeJson({ + const jsonOutput = { root: snapshot.root, note: command.note ?? null, minSupport: command.minSupport, - ...result - }) : sanitizeTerminalText(renderPercolation(result, command.note))); + limit: command.limit, + schemaVersion: result.schemaVersion, + candidates: result.candidates, + truncated: result.truncated + }; + output.stdout(command.json ? terminalSafeJson(jsonOutput) : sanitizeTerminalText(renderPercolation(result, command.note))); return 0; } async function runList(command, output, dependencies) { diff --git a/dist/evaluation-builder.js b/dist/evaluation-builder.js index b60e6dd..9a7aa40 100755 --- a/dist/evaluation-builder.js +++ b/dist/evaluation-builder.js @@ -4,15 +4,15 @@ import { knowledgeBaseEvaluationRetrieverIds, openKnowledgeBaseEvaluation, verifyFrozenEvaluationSnapshot -} from "./index-n5dd7r0v.js"; -import"./index-zzhgcwyt.js"; +} from "./index-xw9ac71d.js"; +import"./index-vxmf14m1.js"; import"./index-adx6khj5.js"; import { indexSemanticVault, recommendedEmbeddingModel, recommendedEmbeddingModelSha256, scanVault -} from "./index-zxdy5pby.js"; +} from "./index-5m2ydj5q.js"; import"./index-4j3tt0c3.js"; import { runGitCommand @@ -22,12 +22,12 @@ import { MAX_EVALUATION_EVIDENCE_BYTES, MAX_EVALUATION_RESULTS_PER_QUERY } from "./index-b88v3vtm.js"; -import"./index-cv6fh7z5.js"; +import"./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; import"./index-5vwpzb5a.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; import"./index-1xxnjn0d.js"; // src/evaluation-builder.ts diff --git a/dist/evaluation-kb.js b/dist/evaluation-kb.js index b231c09..ea98ea3 100644 --- a/dist/evaluation-kb.js +++ b/dist/evaluation-kb.js @@ -4,19 +4,19 @@ import { knowledgeBaseEvaluationRetrieverIds, openKnowledgeBaseEvaluation, verifyFrozenEvaluationSnapshot -} from "./index-n5dd7r0v.js"; -import"./index-zzhgcwyt.js"; +} from "./index-xw9ac71d.js"; +import"./index-vxmf14m1.js"; import"./index-adx6khj5.js"; -import"./index-zxdy5pby.js"; +import"./index-5m2ydj5q.js"; import"./index-4j3tt0c3.js"; import"./index-1gwbassd.js"; import"./index-b88v3vtm.js"; -import"./index-cv6fh7z5.js"; +import"./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; import"./index-5vwpzb5a.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; import"./index-1xxnjn0d.js"; export { verifyFrozenEvaluationSnapshot, diff --git a/dist/graph.js b/dist/graph.js index 9ce89e9..4573f2b 100644 --- a/dist/graph.js +++ b/dist/graph.js @@ -9,6 +9,7 @@ import { catalogEnd, catalogStart, isCanonicalNoteId, + isCanonicalRelationPredicate, lookupNote, metadataValueFromUnknown, normalizeVaultPath, @@ -17,7 +18,7 @@ import { replaceCatalog, searchableMarkdown, wikiLinks -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; export { wikiLinks, searchableMarkdown, @@ -27,6 +28,7 @@ export { normalizeVaultPath, metadataValueFromUnknown, lookupNote, + isCanonicalRelationPredicate, isCanonicalNoteId, catalogStart, catalogEnd, diff --git a/dist/index-zxdy5pby.js b/dist/index-5m2ydj5q.js similarity index 99% rename from dist/index-zxdy5pby.js rename to dist/index-5m2ydj5q.js index 47f24f3..6dc931a 100644 --- a/dist/index-zxdy5pby.js +++ b/dist/index-5m2ydj5q.js @@ -2,7 +2,7 @@ import { fuseRankedCandidates, validateSearchQuery -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; import { MAX_ANALYZED_NOTES, analyzeVault, @@ -12,7 +12,7 @@ import { parseNote, renderCatalog, replaceCatalog -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; // src/semantic.ts import { createHash as createHash2 } from "crypto"; diff --git a/dist/index-dyqwejk5.js b/dist/index-dyqwejk5.js deleted file mode 100644 index 2a361fc..0000000 --- a/dist/index-dyqwejk5.js +++ /dev/null @@ -1,531 +0,0 @@ -// @bun -import { - MAX_ANALYZED_NOTES, - MAX_MENTIONS, - lookupNote -} from "./index-cxfrakt7.js"; - -// src/percolate.ts -import { createHash } from "crypto"; -import { posix } from "path"; -var DEFAULT_PERCOLATION_LIMIT = 100; -var MAX_PERCOLATION_LIMIT = 1000; -var DEFAULT_PERCOLATION_MIN_SUPPORT = 2; -var MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE = 100; -var MAX_PERCOLATION_NOTES = MAX_ANALYZED_NOTES; -var MAX_PERCOLATION_MENTION_PAIRS = 250000; -var MAX_PERCOLATION_MENTIONS = MAX_MENTIONS; -var MAX_SCOPED_PERCOLATION_MENTION_PAIRS = MAX_PERCOLATION_NOTES * 2; -var MAX_PERCOLATION_EVIDENCE = 250000; -var MAX_PERCOLATION_PAIR_OBSERVATIONS = MAX_PERCOLATION_MENTION_PAIRS; -function compareText(left, right) { - return left < right ? -1 : left > right ? 1 : 0; -} -function pairKey(left, right) { - return compareText(left, right) <= 0 ? `${left}\x00${right}` : `${right}\x00${left}`; -} -function directedKey(source, target) { - return `${source}\x00${target}`; -} -function relationKey(source, predicate, target) { - return `${source}\x00${predicate}\x00${target}`; -} -function checkedLine(line, context) { - if (!Number.isSafeInteger(line) || line < 1) { - throw new TypeError(`${context} has an invalid evidence line.`); - } - return line; -} -function checkedOptions(options) { - const limit = options.limit ?? DEFAULT_PERCOLATION_LIMIT; - if (!Number.isSafeInteger(limit) || limit < 0 || limit > MAX_PERCOLATION_LIMIT) { - throw new RangeError(`Percolation limit must be a safe integer from 0 to ${MAX_PERCOLATION_LIMIT}.`); - } - const minSupport = options.minSupport ?? DEFAULT_PERCOLATION_MIN_SUPPORT; - if (!Number.isSafeInteger(minSupport) || minSupport < 1 || minSupport > MAX_PERCOLATION_EVIDENCE) { - throw new RangeError(`Percolation minimum support must be a safe integer from 1 to ${MAX_PERCOLATION_EVIDENCE}.`); - } - return { limit, minSupport }; -} -function indexedContentNotes(notes, analysis) { - if (analysis.noteConnections.length > MAX_PERCOLATION_NOTES) { - throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_NOTES} note limit.`); - } - const allById = new Map; - for (const note of notes) { - if (allById.has(note.id)) { - throw new Error(`Duplicate note identity in percolation: ${note.id}.`); - } - allById.set(note.id, note); - } - const byId = new Map; - const byPath = new Map; - for (const connection of analysis.noteConnections) { - if (byId.has(connection.id)) { - throw new Error(`Duplicate analyzed note identity: ${connection.id}.`); - } - const note = allById.get(connection.id); - if (note === undefined) { - throw new Error(`Analysis references missing note identity: ${connection.id}.`); - } - if (byPath.has(note.path)) { - throw new Error(`Duplicate note path in percolation: ${note.path}.`); - } - byId.set(note.id, note); - byPath.set(note.path, note); - } - return { - notes: [...byId.values()].toSorted((left, right) => compareText(left.id, right.id)), - byId, - byPath - }; -} -function conceptKey(value) { - return value.normalize("NFKC").toLocaleLowerCase("en-US").replace(/^#+/u, "").replace(/[^\p{Letter}\p{Number}]+/gu, "-").replace(/^-+|-+$/gu, ""); -} -function isConcept(note) { - return note.properties.type?.normalize("NFC").toLocaleLowerCase("en-US") === "concept"; -} -function conceptLabels(note) { - return [ - note.title, - ...note.aliases, - posix.basename(note.id) - ].map(conceptKey).filter((value) => value !== ""); -} -function naturalConceptId(tag) { - const slug = conceptKey(tag); - const digest = createHash("sha256").update(tag.normalize("NFC")).digest("hex"); - if (slug === "") - return `notes/concept-${digest.slice(0, 16)}`; - if (slug.length <= 160) - return `notes/${slug}`; - let prefix = ""; - let count = 0; - for (const character of slug) { - if (count >= 144) - break; - prefix += character; - count += 1; - } - return `notes/${prefix.replace(/-+$/u, "")}-${digest.slice(0, 12)}`; -} -function suggestedConceptId(tag, occupiedIds) { - const natural = naturalConceptId(tag); - const foldedNatural = natural.toLocaleLowerCase("en-US"); - const collidesWith = occupiedIds.get(foldedNatural) ?? null; - if (collidesWith === null) - return { id: natural, collidesWith: null }; - const suffixed = `${natural}-concept`; - if (!occupiedIds.has(suffixed.toLocaleLowerCase("en-US"))) { - return { id: suffixed, collidesWith }; - } - for (let suffix = 2;suffix <= MAX_PERCOLATION_NOTES + 2; suffix += 1) { - const candidate = `${suffixed}-${suffix}`; - if (!occupiedIds.has(candidate.toLocaleLowerCase("en-US"))) { - return { id: candidate, collidesWith }; - } - } - throw new RangeError("Percolation could not choose an unoccupied concept ID."); -} -function resolvedNoteFilter(notes, query) { - if (query === undefined) - return null; - const result = lookupNote(notes, query); - if (result.kind === "found") - return result.note.id; - if (result.kind === "ambiguous") { - throw new Error(`Percolation note is ambiguous: ${result.candidates.map((note) => note.id).join(", ")}.`); - } - throw new Error(`Percolation note does not exist: ${query}.`); -} -function candidateInvolvesNote(candidate, note) { - if (note === null) - return true; - if (candidate.kind === "missing-concept") { - return candidate.evidence.some((evidence) => evidence.note === note); - } - return candidate.source === note || candidate.target === note; -} -function compareSharedEvidence(left, right) { - return compareText(left.kind, right.kind) || compareText(left.kind === "shared-tag" ? left.tag : left.concept, right.kind === "shared-tag" ? right.tag : right.concept) || compareText(left.note, right.note); -} -function candidateIdentity(candidate) { - switch (candidate.kind) { - case "missing-concept": - return candidate.tag; - case "missing-relation": - return `${candidate.source}\x00${candidate.target}`; - case "unlinked-mention": - return `${candidate.source}\x00${candidate.target}`; - case "relation-hygiene": - return [ - candidate.problem, - candidate.source, - candidate.predicate ?? "", - candidate.target ?? "" - ].join("\x00"); - } -} -var candidateKindRank = { - "relation-hygiene": 0, - "unlinked-mention": 1, - "missing-relation": 2, - "missing-concept": 3 -}; -function compareCandidates(left, right) { - return right.support - left.support || candidateKindRank[left.kind] - candidateKindRank[right.kind] || compareText(candidateIdentity(left), candidateIdentity(right)); -} -function relationEvidence(relation) { - return { - kind: "relation", - source: relation.source, - target: relation.target, - predicate: relation.predicate, - line: checkedLine(relation.provenance.line, `Authored relation ${relation.source} -> ${relation.target}`), - authoredTarget: relation.provenance.authoredTarget - }; -} -function issueEvidence(issue, source) { - const line = checkedLine(issue.line, `Relation issue in ${issue.source}`); - if (issue.kind === "malformed") { - return { - kind: "relation-issue", - issue: issue.kind, - source, - line, - predicate: issue.predicate ?? null, - target: issue.target ?? null, - candidates: [], - candidatesTruncated: false, - message: issue.message - }; - } - if (issue.kind === "broken") { - return { - kind: "relation-issue", - issue: issue.kind, - source, - line, - predicate: issue.predicate, - target: issue.target, - candidates: [], - candidatesTruncated: false, - message: `Relationship target does not exist: ${issue.target}.` - }; - } - return { - kind: "relation-issue", - issue: issue.kind, - source, - line, - predicate: issue.predicate, - target: issue.target, - candidates: [...issue.candidates].toSorted(compareText).slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE), - candidatesTruncated: issue.candidates.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, - message: `Relationship target is ambiguous: ${issue.target}.` - }; -} -function percolateVault(notes, analysis, options = {}) { - const { limit, minSupport } = checkedOptions(options); - const indexed = indexedContentNotes(notes, analysis); - const noteFilter = resolvedNoteFilter(indexed.notes, options.note); - const relations = analysis.authoredRelations ?? []; - const relationIssues = analysis.relationIssues ?? []; - let evidenceObservations = analysis.mentions.length + relations.length + relationIssues.length; - for (const issue of relationIssues) { - if (issue.kind !== "ambiguous") - continue; - evidenceObservations += issue.candidates.length; - if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) - break; - } - if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) { - throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} evidence limit.`); - } - const conceptIds = new Set(indexed.notes.filter(isConcept).map((note) => note.id)); - const occupiedIds = new Map(indexed.notes.map((note) => [ - note.id.toLocaleLowerCase("en-US"), - note.id - ])); - const nonConceptNotes = indexed.notes.filter((note) => !conceptIds.has(note.id)); - const conceptLabelKeys = new Set(indexed.notes.filter((note) => conceptIds.has(note.id)).flatMap(conceptLabels)); - const candidates = []; - const notesByTag = new Map; - let tagEvidenceCount = 0; - for (const note of nonConceptNotes) { - for (const tag of new Set(note.tags)) { - tagEvidenceCount += 1; - if (tagEvidenceCount > MAX_PERCOLATION_EVIDENCE) { - throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} tag evidence limit.`); - } - const matches = notesByTag.get(tag) ?? []; - matches.push(note); - notesByTag.set(tag, matches); - } - } - for (const [tag, matchingNotes] of [...notesByTag].toSorted(([left], [right]) => compareText(left, right))) { - const sortedMatches = matchingNotes.toSorted((left, right) => (left.id === noteFilter ? -1 : 0) - (right.id === noteFilter ? -1 : 0) || compareText(left.id, right.id)); - const evidence = sortedMatches.slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE).map((note) => ({ - kind: "tag", - note: note.id, - path: note.path, - tag - })); - if (matchingNotes.length >= Math.max(2, minSupport) && !conceptLabelKeys.has(conceptKey(tag)) && (noteFilter === null || matchingNotes.some((note) => note.id === noteFilter))) { - const suggestion = suggestedConceptId(tag, occupiedIds); - candidates.push({ - kind: "missing-concept", - tag, - suggestedId: suggestion.id, - collidesWith: suggestion.collidesWith, - support: matchingNotes.length, - evidenceTruncated: matchingNotes.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, - evidence - }); - } - } - const explicitPairs = new Set; - const noteConcepts = new Map; - const addConceptConnection = (noteId, conceptId) => { - if (conceptIds.has(noteId) || !conceptIds.has(conceptId)) - return; - const concepts = noteConcepts.get(noteId) ?? new Set; - concepts.add(conceptId); - noteConcepts.set(noteId, concepts); - }; - for (const link of analysis.contextualLinks) { - const source = indexed.byPath.get(link.source); - const target = indexed.byPath.get(link.target); - if (source === undefined || target === undefined) { - throw new Error(`Contextual link references an unknown percolation note: ${link.source} -> ${link.target}.`); - } - explicitPairs.add(pairKey(source.id, target.id)); - addConceptConnection(source.id, target.id); - addConceptConnection(target.id, source.id); - } - for (const relation of relations) { - if (!indexed.byId.has(relation.source) || !indexed.byId.has(relation.target)) { - throw new Error(`Authored relation references an unknown percolation note: ${relation.source} -> ${relation.target}.`); - } - explicitPairs.add(pairKey(relation.source, relation.target)); - addConceptConnection(relation.source, relation.target); - addConceptConnection(relation.target, relation.source); - } - const pairEvidence = new Map; - let pairObservations = 0; - const addPairEvidence = (left, right, evidence) => { - pairObservations += 1; - if (pairObservations > MAX_PERCOLATION_PAIR_OBSERVATIONS) { - throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_PAIR_OBSERVATIONS} pair observation limit.`); - } - const key = pairKey(left.id, right.id); - if (explicitPairs.has(key)) - return; - const accumulated = pairEvidence.get(key) ?? { - support: 0, - evidenceCount: 0, - evidence: [] - }; - accumulated.support += 1; - accumulated.evidenceCount += evidence.length; - for (const item of evidence) { - if (accumulated.evidence.length < MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE) { - accumulated.evidence.push(item); - } - } - pairEvidence.set(key, accumulated); - }; - const addPairsForGroup = (matchingNotes, evidenceFor) => { - const sortedNotes = matchingNotes.toSorted((left, right) => compareText(left.id, right.id)); - if (noteFilter !== null) { - const scoped = sortedNotes.find((note) => note.id === noteFilter); - if (scoped === undefined) - return; - for (const other of sortedNotes) { - if (other.id === scoped.id) - continue; - const [left, right] = compareText(scoped.id, other.id) < 0 ? [scoped, other] : [other, scoped]; - addPairEvidence(left, right, evidenceFor(left, right)); - } - return; - } - for (let leftIndex = 0;leftIndex < sortedNotes.length; leftIndex += 1) { - for (let rightIndex = leftIndex + 1;rightIndex < sortedNotes.length; rightIndex += 1) { - const left = sortedNotes[leftIndex]; - const right = sortedNotes[rightIndex]; - if (left === undefined || right === undefined) - continue; - addPairEvidence(left, right, evidenceFor(left, right)); - } - } - }; - for (const [tag, matchingNotes] of [...notesByTag].toSorted(([left], [right]) => compareText(left, right))) { - addPairsForGroup(matchingNotes, (left, right) => [ - { kind: "shared-tag", note: left.id, path: left.path, tag }, - { kind: "shared-tag", note: right.id, path: right.path, tag } - ]); - } - const notesByConcept = new Map; - for (const [noteId, connectedConcepts] of noteConcepts) { - const note = indexed.byId.get(noteId); - if (note === undefined) - continue; - for (const concept of connectedConcepts) { - const matches = notesByConcept.get(concept) ?? []; - matches.push(note); - notesByConcept.set(concept, matches); - } - } - for (const [concept, matchingNotes] of [...notesByConcept].toSorted(([left], [right]) => compareText(left, right))) { - const conceptNote = indexed.byId.get(concept); - if (conceptNote === undefined) - continue; - addPairsForGroup(matchingNotes, (left, right) => [ - { - kind: "shared-concept", - note: left.id, - path: left.path, - concept, - conceptPath: conceptNote.path - }, - { - kind: "shared-concept", - note: right.id, - path: right.path, - concept, - conceptPath: conceptNote.path - } - ]); - } - for (const [key, accumulated] of pairEvidence) { - const separator = key.indexOf("\x00"); - const source = key.slice(0, separator); - const target = key.slice(separator + 1); - const evidence = accumulated.evidence.toSorted(compareSharedEvidence); - if (accumulated.support < minSupport) - continue; - candidates.push({ - kind: "missing-relation", - source, - target, - suggestedPredicate: "related-to", - support: accumulated.support, - evidenceTruncated: accumulated.evidenceCount > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, - evidence - }); - } - const mentionEvidenceByPair = new Map; - for (const mention of analysis.mentions) { - const source = indexed.byPath.get(mention.source); - const target = indexed.byPath.get(mention.target); - if (source === undefined || target === undefined) { - throw new Error(`Mention references an unknown percolation note: ${mention.source} -> ${mention.target}.`); - } - if (noteFilter !== null && source.id !== noteFilter && target.id !== noteFilter) - continue; - if (explicitPairs.has(pairKey(source.id, target.id))) - continue; - const key = directedKey(source.id, target.id); - const accumulated = mentionEvidenceByPair.get(key) ?? { - support: 0, - evidence: [] - }; - accumulated.support += 1; - if (accumulated.evidence.length < MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE) - accumulated.evidence.push({ - kind: "mention", - source: source.id, - target: target.id, - line: checkedLine(mention.line, `Mention ${mention.source} -> ${mention.target}`), - phrase: mention.phrase - }); - mentionEvidenceByPair.set(key, accumulated); - } - for (const [key, accumulated] of mentionEvidenceByPair) { - const separator = key.indexOf("\x00"); - const source = key.slice(0, separator); - const target = key.slice(separator + 1); - const sortedEvidence = accumulated.evidence.toSorted((left, right) => left.line - right.line || compareText(left.phrase, right.phrase)); - candidates.push({ - kind: "unlinked-mention", - source, - target, - support: accumulated.support, - evidenceTruncated: accumulated.support > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, - evidence: sortedEvidence - }); - } - const relationsByKey = new Map(relations.map((relation) => [ - relationKey(relation.source, relation.predicate, relation.target), - relation - ])); - for (const relation of relations) { - if (noteFilter !== null && relation.source !== noteFilter && relation.target !== noteFilter) - continue; - if (relation.source === relation.target) { - candidates.push({ - kind: "relation-hygiene", - problem: "self-relation", - source: relation.source, - target: relation.target, - predicate: relation.predicate, - message: "Review an authored relationship whose source and target are the same note.", - support: 1, - evidenceTruncated: false, - evidence: [relationEvidence(relation)] - }); - continue; - } - if (compareText(relation.source, relation.target) >= 0) - continue; - const reciprocal = relationsByKey.get(relationKey(relation.target, relation.predicate, relation.source)); - if (reciprocal === undefined) - continue; - const evidence = [ - relationEvidence(relation), - relationEvidence(reciprocal) - ].toSorted((left, right) => compareText(left.source, right.source) || compareText(left.target, right.target)); - candidates.push({ - kind: "relation-hygiene", - problem: "reciprocal-relation", - source: relation.source, - target: relation.target, - predicate: relation.predicate, - message: "Review reciprocal assertions of the same directional predicate.", - support: evidence.length, - evidenceTruncated: false, - evidence - }); - } - for (const issue of relationIssues) { - const sourceNote = indexed.byPath.get(issue.source); - if (sourceNote === undefined) - continue; - if (noteFilter !== null && sourceNote.id !== noteFilter) - continue; - const evidence = issueEvidence(issue, sourceNote.id); - const problem = issue.kind === "malformed" ? "malformed-relation" : issue.kind === "broken" ? "broken-relation" : "ambiguous-relation"; - candidates.push({ - kind: "relation-hygiene", - problem, - source: sourceNote.id, - target: evidence.target, - predicate: evidence.predicate, - message: evidence.message, - support: 1, - evidenceTruncated: evidence.candidatesTruncated, - evidence: [evidence] - }); - } - if (candidates.length > MAX_PERCOLATION_EVIDENCE) { - throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} candidate limit.`); - } - const sorted = candidates.filter((candidate) => candidateInvolvesNote(candidate, noteFilter)).toSorted(compareCandidates); - return { - candidates: sorted.slice(0, limit), - truncated: sorted.length > limit - }; -} - -export { DEFAULT_PERCOLATION_LIMIT, MAX_PERCOLATION_LIMIT, DEFAULT_PERCOLATION_MIN_SUPPORT, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, MAX_PERCOLATION_NOTES, MAX_PERCOLATION_MENTION_PAIRS, MAX_PERCOLATION_MENTIONS, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, percolateVault }; diff --git a/dist/index-cxfrakt7.js b/dist/index-ekpwvbra.js similarity index 99% rename from dist/index-cxfrakt7.js rename to dist/index-ekpwvbra.js index 28dca2a..6ad3a22 100644 --- a/dist/index-cxfrakt7.js +++ b/dist/index-ekpwvbra.js @@ -207,6 +207,9 @@ function emptyMetadata() { } var relationPredicatePattern = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u; var maxNoteIdLength = 2048; +function isCanonicalRelationPredicate(value) { + return value !== "" && value === value.normalize("NFC") && relationPredicatePattern.test(value); +} function isCanonicalNoteId(value) { if (value === "" || value.length > maxNoteIdLength || value !== value.trim() || value !== value.normalize("NFC") || value.includes("\\") || value.includes("\x00") || value.includes(` `) || value.includes("\r") || value.startsWith("/") || value.endsWith("/") || value.toLocaleLowerCase("en-US").endsWith(".md")) { @@ -275,7 +278,7 @@ function parsedRelations(contents, source, lineCounter) { continue; } const predicate = pair.key.value.normalize("NFC"); - if (!relationPredicatePattern.test(predicate)) { + if (!isCanonicalRelationPredicate(predicate)) { relationIssues.push(malformedRelation(source, predicateLine, `Invalid relation predicate "${predicate}"; use strict lower kebab-case.`, { predicate })); continue; } @@ -1064,4 +1067,4 @@ function replaceCatalog(indexContent, catalog) { return indexContent.slice(0, start) + catalog + indexContent.slice(end + catalogEnd.length); } -export { MAX_PORTFOLIO_NAME_BYTES, MAX_DOCUMENT_ID_BYTES, portfolioVaultIdentity, parseVaultKey, parseDocumentId, documentIdState, formatQualifiedDocumentUri, parseQualifiedDocumentUri, portfolioDocumentIdentity, catalogStart, catalogEnd, MAX_ANALYZED_NOTES, MAX_CONNECTION_OBSERVATIONS, MAX_MENTION_PAIRS, MAX_MENTIONS, VaultAnalysisBudgetError, metadataValueFromUnknown, isCanonicalNoteId, normalizeVaultPath, searchableMarkdown, wikiLinks, parseNote, lookupNote, analyzeVault, renderCatalog, replaceCatalog }; +export { MAX_PORTFOLIO_NAME_BYTES, MAX_DOCUMENT_ID_BYTES, portfolioVaultIdentity, parseVaultKey, parseDocumentId, documentIdState, formatQualifiedDocumentUri, parseQualifiedDocumentUri, portfolioDocumentIdentity, catalogStart, catalogEnd, MAX_ANALYZED_NOTES, MAX_CONNECTION_OBSERVATIONS, MAX_MENTION_PAIRS, MAX_MENTIONS, VaultAnalysisBudgetError, metadataValueFromUnknown, isCanonicalRelationPredicate, isCanonicalNoteId, normalizeVaultPath, searchableMarkdown, wikiLinks, parseNote, lookupNote, analyzeVault, renderCatalog, replaceCatalog }; diff --git a/dist/index-jsmvyyvf.js b/dist/index-ey46z1zf.js similarity index 99% rename from dist/index-jsmvyyvf.js rename to dist/index-ey46z1zf.js index 4fd8243..03d6cad 100644 --- a/dist/index-jsmvyyvf.js +++ b/dist/index-ey46z1zf.js @@ -5,14 +5,14 @@ import { MAX_SEARCH_RESULTS, openKnowledgeBase, validateKnowledgeBaseSearchHistory -} from "./index-zzhgcwyt.js"; +} from "./index-vxmf14m1.js"; import { expandSearchRequest, parseSearchRules } from "./index-adx6khj5.js"; import { scanVault -} from "./index-zxdy5pby.js"; +} from "./index-5m2ydj5q.js"; import { indexGitHistory } from "./index-1gwbassd.js"; @@ -22,7 +22,7 @@ import { } from "./index-x3fthpsc.js"; import { validateSearchQuery -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; import { validateQueryOptions } from "./index-48pz4jpc.js"; @@ -32,7 +32,7 @@ import { parseVaultKey, portfolioDocumentIdentity, portfolioVaultIdentity -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; // src/portfolio.ts import { createHash as createHash2 } from "crypto"; diff --git a/dist/index-f9fy4w1n.js b/dist/index-f9fy4w1n.js new file mode 100644 index 0000000..b568c02 --- /dev/null +++ b/dist/index-f9fy4w1n.js @@ -0,0 +1,1121 @@ +// @bun +import { + MAX_ANALYZED_NOTES, + MAX_MENTIONS, + isCanonicalNoteId, + isCanonicalRelationPredicate, + lookupNote +} from "./index-ekpwvbra.js"; + +// src/percolate.ts +import { createHash } from "crypto"; +import { posix } from "path"; +var DEFAULT_PERCOLATION_LIMIT = 100; +var MAX_PERCOLATION_LIMIT = 1000; +var DEFAULT_PERCOLATION_MIN_SUPPORT = 2; +var MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE = 100; +var MAX_PERCOLATION_NOTES = MAX_ANALYZED_NOTES; +var MAX_PERCOLATION_MENTION_PAIRS = 250000; +var MAX_PERCOLATION_MENTIONS = MAX_MENTIONS; +var MAX_SCOPED_PERCOLATION_MENTION_PAIRS = MAX_PERCOLATION_NOTES * 2; +var PERCOLATION_RESULT_SCHEMA_VERSION = 2; +var MAX_PERCOLATION_RESULT_NODES = 250000; +var MAX_PERCOLATION_RESULT_UTF8_BYTES = 16 * 1024 * 1024; +var MAX_PERCOLATION_TEXT_UTF8_BYTES = 64 * 1024; +var MAX_PERCOLATION_EVIDENCE = 250000; +var MAX_PERCOLATION_PAIR_OBSERVATIONS = MAX_PERCOLATION_MENTION_PAIRS; +function compareText(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} +function pairKey(left, right) { + return compareText(left, right) <= 0 ? `${left}\x00${right}` : `${right}\x00${left}`; +} +function directedKey(source, target) { + return `${source}\x00${target}`; +} +function relationKey(source, predicate, target) { + return `${source}\x00${predicate}\x00${target}`; +} +function checkedLine(line, context) { + if (!Number.isSafeInteger(line) || line < 1) { + throw new TypeError(`${context} has an invalid evidence line.`); + } + return line; +} +function checkedOptions(options) { + const limit = options.limit ?? DEFAULT_PERCOLATION_LIMIT; + if (!Number.isSafeInteger(limit) || limit < 0 || limit > MAX_PERCOLATION_LIMIT) { + throw new RangeError(`Percolation limit must be a safe integer from 0 to ${MAX_PERCOLATION_LIMIT}.`); + } + const minSupport = options.minSupport ?? DEFAULT_PERCOLATION_MIN_SUPPORT; + if (!Number.isSafeInteger(minSupport) || minSupport < 1 || minSupport > MAX_PERCOLATION_EVIDENCE) { + throw new RangeError(`Percolation minimum support must be a safe integer from 1 to ${MAX_PERCOLATION_EVIDENCE}.`); + } + return { limit, minSupport }; +} +function indexedContentNotes(notes, analysis) { + if (analysis.noteConnections.length > MAX_PERCOLATION_NOTES) { + throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_NOTES} note limit.`); + } + const allById = new Map; + for (const note of notes) { + if (allById.has(note.id)) { + throw new Error(`Duplicate note identity in percolation: ${note.id}.`); + } + allById.set(note.id, note); + } + const byId = new Map; + const byPath = new Map; + for (const connection of analysis.noteConnections) { + if (byId.has(connection.id)) { + throw new Error(`Duplicate analyzed note identity: ${connection.id}.`); + } + const note = allById.get(connection.id); + if (note === undefined) { + throw new Error(`Analysis references missing note identity: ${connection.id}.`); + } + if (byPath.has(note.path)) { + throw new Error(`Duplicate note path in percolation: ${note.path}.`); + } + byId.set(note.id, note); + byPath.set(note.path, note); + } + return { + notes: [...byId.values()].toSorted((left, right) => compareText(left.id, right.id)), + byId, + byPath + }; +} +function conceptKey(value) { + return value.normalize("NFKC").toLocaleLowerCase("en-US").replace(/^#+/u, "").replace(/[^\p{Letter}\p{Number}]+/gu, "-").replace(/^-+|-+$/gu, ""); +} +function isConcept(note) { + return note.properties.type?.normalize("NFC").toLocaleLowerCase("en-US") === "concept"; +} +function conceptLabels(note) { + return [ + note.title, + ...note.aliases, + posix.basename(note.id) + ].map(conceptKey).filter((value) => value !== ""); +} +function naturalConceptId(tag) { + const slug = conceptKey(tag); + const digest = createHash("sha256").update(tag.normalize("NFC")).digest("hex"); + if (slug === "") + return `notes/concept-${digest.slice(0, 16)}`; + if (slug.length <= 160) + return `notes/${slug}`; + let prefix = ""; + let count = 0; + for (const character of slug) { + if (count >= 144) + break; + prefix += character; + count += 1; + } + return `notes/${prefix.replace(/-+$/u, "")}-${digest.slice(0, 12)}`; +} +function suggestedConceptId(tag, occupiedIds) { + const natural = naturalConceptId(tag); + const foldedNatural = natural.toLocaleLowerCase("en-US"); + const collidesWith = occupiedIds.get(foldedNatural) ?? null; + if (collidesWith === null) + return { id: natural, collidesWith: null }; + const suffixed = `${natural}-concept`; + if (!occupiedIds.has(suffixed.toLocaleLowerCase("en-US"))) { + return { id: suffixed, collidesWith }; + } + for (let suffix = 2;suffix <= MAX_PERCOLATION_NOTES + 2; suffix += 1) { + const candidate = `${suffixed}-${suffix}`; + if (!occupiedIds.has(candidate.toLocaleLowerCase("en-US"))) { + return { id: candidate, collidesWith }; + } + } + throw new RangeError("Percolation could not choose an unoccupied concept ID."); +} +function resolvedNoteFilter(notes, query) { + if (query === undefined) + return null; + const result = lookupNote(notes, query); + if (result.kind === "found") + return result.note.id; + if (result.kind === "ambiguous") { + throw new Error(`Percolation note is ambiguous: ${result.candidates.map((note) => note.id).join(", ")}.`); + } + throw new Error(`Percolation note does not exist: ${query}.`); +} +function candidateInvolvesNote(candidate, note) { + if (note === null) + return true; + if (candidate.kind === "missing-concept") { + return candidate.evidence.some((evidence) => evidence.note === note); + } + return candidate.source === note || candidate.target === note; +} +function compareSharedEvidence(left, right) { + return compareText(left.kind, right.kind) || compareText(left.kind === "shared-tag" ? left.tag : left.concept, right.kind === "shared-tag" ? right.tag : right.concept) || compareText(left.note, right.note); +} +function candidateIdentity(candidate) { + switch (candidate.kind) { + case "missing-concept": + return candidate.tag; + case "missing-relation": + return `${candidate.source}\x00${candidate.target}`; + case "unlinked-mention": + return `${candidate.source}\x00${candidate.target}`; + case "relation-hygiene": + return [ + candidate.problem, + candidate.source, + candidate.predicate ?? "", + candidate.target ?? "", + candidate.message, + String(candidate.evidence[0]?.line ?? 0) + ].join("\x00"); + } +} +var candidateKindRank = { + "relation-hygiene": 0, + "unlinked-mention": 1, + "missing-relation": 2, + "missing-concept": 3 +}; +function compareCandidates(left, right) { + return right.support - left.support || candidateKindRank[left.kind] - candidateKindRank[right.kind] || compareText(candidateIdentity(left), candidateIdentity(right)); +} +function relationEvidence(relation) { + return { + kind: "relation", + source: relation.source, + target: relation.target, + predicate: relation.predicate, + line: checkedLine(relation.provenance.line, `Authored relation ${relation.source} -> ${relation.target}`), + authoredTarget: relation.provenance.authoredTarget + }; +} +function issueEvidence(issue, source) { + const line = checkedLine(issue.line, `Relation issue in ${issue.source}`); + if (issue.kind === "malformed") { + return { + kind: "relation-issue", + issue: issue.kind, + source, + line, + predicate: issue.predicate ?? null, + target: issue.target ?? null, + candidates: [], + candidatesTruncated: false, + message: issue.message + }; + } + if (issue.kind === "broken") { + return { + kind: "relation-issue", + issue: issue.kind, + source, + line, + predicate: issue.predicate, + target: issue.target, + candidates: [], + candidatesTruncated: false, + message: `Relationship target does not exist: ${issue.target}.` + }; + } + return { + kind: "relation-issue", + issue: issue.kind, + source, + line, + predicate: issue.predicate, + target: issue.target, + candidates: [...issue.candidates].toSorted(compareText).slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE), + candidatesTruncated: issue.candidates.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + message: `Relationship target is ambiguous: ${issue.target}.` + }; +} +function percolateVault(notes, analysis, options = {}) { + const { limit, minSupport } = checkedOptions(options); + const indexed = indexedContentNotes(notes, analysis); + const noteFilter = resolvedNoteFilter(indexed.notes, options.note); + const relations = analysis.authoredRelations ?? []; + const relationIssues = analysis.relationIssues ?? []; + let evidenceObservations = analysis.mentions.length + relations.length + relationIssues.length; + for (const issue of relationIssues) { + if (issue.kind !== "ambiguous") + continue; + evidenceObservations += issue.candidates.length; + if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) + break; + } + if (evidenceObservations > MAX_PERCOLATION_EVIDENCE) { + throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} evidence limit.`); + } + const conceptIds = new Set(indexed.notes.filter(isConcept).map((note) => note.id)); + const occupiedIds = new Map(indexed.notes.map((note) => [ + note.id.toLocaleLowerCase("en-US"), + note.id + ])); + const nonConceptNotes = indexed.notes.filter((note) => !conceptIds.has(note.id)); + const conceptLabelKeys = new Set(indexed.notes.filter((note) => conceptIds.has(note.id)).flatMap(conceptLabels)); + const candidates = []; + const notesByTag = new Map; + let tagEvidenceCount = 0; + for (const note of nonConceptNotes) { + for (const tag of new Set(note.tags)) { + tagEvidenceCount += 1; + if (tagEvidenceCount > MAX_PERCOLATION_EVIDENCE) { + throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} tag evidence limit.`); + } + const matches = notesByTag.get(tag) ?? []; + matches.push(note); + notesByTag.set(tag, matches); + } + } + for (const [tag, matchingNotes] of [...notesByTag].toSorted(([left], [right]) => compareText(left, right))) { + const sortedMatches = matchingNotes.toSorted((left, right) => (left.id === noteFilter ? -1 : 0) - (right.id === noteFilter ? -1 : 0) || compareText(left.id, right.id)); + const evidence = sortedMatches.slice(0, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE).map((note) => ({ + kind: "tag", + note: note.id, + path: note.path, + tag + })); + if (matchingNotes.length >= Math.max(2, minSupport) && !conceptLabelKeys.has(conceptKey(tag)) && (noteFilter === null || matchingNotes.some((note) => note.id === noteFilter))) { + const suggestion = suggestedConceptId(tag, occupiedIds); + candidates.push({ + kind: "missing-concept", + tag, + suggestedId: suggestion.id, + collidesWith: suggestion.collidesWith, + support: matchingNotes.length, + evidenceTruncated: matchingNotes.length > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + evidence + }); + } + } + const explicitPairs = new Set; + const noteConcepts = new Map; + const addConceptConnection = (noteId, conceptId) => { + if (conceptIds.has(noteId) || !conceptIds.has(conceptId)) + return; + const concepts = noteConcepts.get(noteId) ?? new Set; + concepts.add(conceptId); + noteConcepts.set(noteId, concepts); + }; + for (const link of analysis.contextualLinks) { + const source = indexed.byPath.get(link.source); + const target = indexed.byPath.get(link.target); + if (source === undefined || target === undefined) { + throw new Error(`Contextual link references an unknown percolation note: ${link.source} -> ${link.target}.`); + } + explicitPairs.add(pairKey(source.id, target.id)); + addConceptConnection(source.id, target.id); + addConceptConnection(target.id, source.id); + } + for (const relation of relations) { + if (!indexed.byId.has(relation.source) || !indexed.byId.has(relation.target)) { + throw new Error(`Authored relation references an unknown percolation note: ${relation.source} -> ${relation.target}.`); + } + explicitPairs.add(pairKey(relation.source, relation.target)); + addConceptConnection(relation.source, relation.target); + addConceptConnection(relation.target, relation.source); + } + const pairEvidence = new Map; + let pairObservations = 0; + const addPairEvidence = (left, right, evidence) => { + pairObservations += 1; + if (pairObservations > MAX_PERCOLATION_PAIR_OBSERVATIONS) { + throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_PAIR_OBSERVATIONS} pair observation limit.`); + } + const key = pairKey(left.id, right.id); + if (explicitPairs.has(key)) + return; + const accumulated = pairEvidence.get(key) ?? { + support: 0, + evidenceCount: 0, + evidence: [] + }; + accumulated.support += 1; + accumulated.evidenceCount += evidence.length; + for (const item of evidence) { + if (accumulated.evidence.length < MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE) { + accumulated.evidence.push(item); + } + } + pairEvidence.set(key, accumulated); + }; + const addPairsForGroup = (matchingNotes, evidenceFor) => { + const sortedNotes = matchingNotes.toSorted((left, right) => compareText(left.id, right.id)); + if (noteFilter !== null) { + const scoped = sortedNotes.find((note) => note.id === noteFilter); + if (scoped === undefined) + return; + for (const other of sortedNotes) { + if (other.id === scoped.id) + continue; + const [left, right] = compareText(scoped.id, other.id) < 0 ? [scoped, other] : [other, scoped]; + addPairEvidence(left, right, evidenceFor(left, right)); + } + return; + } + for (let leftIndex = 0;leftIndex < sortedNotes.length; leftIndex += 1) { + for (let rightIndex = leftIndex + 1;rightIndex < sortedNotes.length; rightIndex += 1) { + const left = sortedNotes[leftIndex]; + const right = sortedNotes[rightIndex]; + if (left === undefined || right === undefined) + continue; + addPairEvidence(left, right, evidenceFor(left, right)); + } + } + }; + for (const [tag, matchingNotes] of [...notesByTag].toSorted(([left], [right]) => compareText(left, right))) { + addPairsForGroup(matchingNotes, (left, right) => [ + { kind: "shared-tag", note: left.id, path: left.path, tag }, + { kind: "shared-tag", note: right.id, path: right.path, tag } + ]); + } + const notesByConcept = new Map; + for (const [noteId, connectedConcepts] of noteConcepts) { + const note = indexed.byId.get(noteId); + if (note === undefined) + continue; + for (const concept of connectedConcepts) { + const matches = notesByConcept.get(concept) ?? []; + matches.push(note); + notesByConcept.set(concept, matches); + } + } + for (const [concept, matchingNotes] of [...notesByConcept].toSorted(([left], [right]) => compareText(left, right))) { + const conceptNote = indexed.byId.get(concept); + if (conceptNote === undefined) + continue; + addPairsForGroup(matchingNotes, (left, right) => [ + { + kind: "shared-concept", + note: left.id, + path: left.path, + concept, + conceptPath: conceptNote.path + }, + { + kind: "shared-concept", + note: right.id, + path: right.path, + concept, + conceptPath: conceptNote.path + } + ]); + } + for (const [key, accumulated] of pairEvidence) { + const separator = key.indexOf("\x00"); + const source = key.slice(0, separator); + const target = key.slice(separator + 1); + const evidence = accumulated.evidence.toSorted(compareSharedEvidence); + if (accumulated.support < minSupport) + continue; + candidates.push({ + kind: "missing-relation", + source, + target, + predicate: { kind: "required" }, + support: accumulated.support, + evidenceTruncated: accumulated.evidenceCount > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + evidence + }); + } + const mentionEvidenceByPair = new Map; + for (const mention of analysis.mentions) { + const source = indexed.byPath.get(mention.source); + const target = indexed.byPath.get(mention.target); + if (source === undefined || target === undefined) { + throw new Error(`Mention references an unknown percolation note: ${mention.source} -> ${mention.target}.`); + } + if (noteFilter !== null && source.id !== noteFilter && target.id !== noteFilter) + continue; + if (explicitPairs.has(pairKey(source.id, target.id))) + continue; + const key = directedKey(source.id, target.id); + const accumulated = mentionEvidenceByPair.get(key) ?? { + support: 0, + evidence: [] + }; + accumulated.support += 1; + if (accumulated.evidence.length < MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE) + accumulated.evidence.push({ + kind: "mention", + source: source.id, + target: target.id, + line: checkedLine(mention.line, `Mention ${mention.source} -> ${mention.target}`), + phrase: mention.phrase + }); + mentionEvidenceByPair.set(key, accumulated); + } + for (const [key, accumulated] of mentionEvidenceByPair) { + const separator = key.indexOf("\x00"); + const source = key.slice(0, separator); + const target = key.slice(separator + 1); + const sortedEvidence = accumulated.evidence.toSorted((left, right) => left.line - right.line || compareText(left.phrase, right.phrase)); + candidates.push({ + kind: "unlinked-mention", + source, + target, + support: accumulated.support, + evidenceTruncated: accumulated.support > MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, + evidence: sortedEvidence + }); + } + const relationsByKey = new Map(relations.map((relation) => [ + relationKey(relation.source, relation.predicate, relation.target), + relation + ])); + for (const relation of relations) { + if (noteFilter !== null && relation.source !== noteFilter && relation.target !== noteFilter) + continue; + if (relation.source === relation.target) { + candidates.push({ + kind: "relation-hygiene", + problem: "self-relation", + source: relation.source, + target: relation.target, + predicate: relation.predicate, + message: "Review an authored relationship whose source and target are the same note.", + support: 1, + evidenceTruncated: false, + evidence: [relationEvidence(relation)] + }); + continue; + } + if (compareText(relation.source, relation.target) >= 0) + continue; + const reciprocal = relationsByKey.get(relationKey(relation.target, relation.predicate, relation.source)); + if (reciprocal === undefined) + continue; + const evidence = [ + relationEvidence(relation), + relationEvidence(reciprocal) + ].toSorted((left, right) => compareText(left.source, right.source) || compareText(left.target, right.target)); + candidates.push({ + kind: "relation-hygiene", + problem: "reciprocal-relation", + source: relation.source, + target: relation.target, + predicate: relation.predicate, + message: "Review reciprocal assertions of the same directional predicate.", + support: evidence.length, + evidenceTruncated: false, + evidence + }); + } + for (const issue of relationIssues) { + const sourceNote = indexed.byPath.get(issue.source); + if (sourceNote === undefined) + continue; + if (noteFilter !== null && sourceNote.id !== noteFilter) + continue; + const evidence = issueEvidence(issue, sourceNote.id); + const problem = issue.kind === "malformed" ? "malformed-relation" : issue.kind === "broken" ? "broken-relation" : "ambiguous-relation"; + candidates.push({ + kind: "relation-hygiene", + problem, + source: sourceNote.id, + target: evidence.target, + predicate: evidence.predicate, + message: evidence.message, + support: 1, + evidenceTruncated: evidence.candidatesTruncated, + evidence: [evidence] + }); + } + if (candidates.length > MAX_PERCOLATION_EVIDENCE) { + throw new RangeError(`Percolation exceeds the ${MAX_PERCOLATION_EVIDENCE} candidate limit.`); + } + const uniqueCandidates = new Map; + for (const candidate of candidates) { + const identity = `${candidate.kind}\x00${candidateIdentity(candidate)}`; + if (!uniqueCandidates.has(identity)) + uniqueCandidates.set(identity, candidate); + } + const sorted = [...uniqueCandidates.values()].filter((candidate) => candidateInvolvesNote(candidate, noteFilter)).toSorted(compareCandidates); + return parsePercolationResultV2({ + schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION, + candidates: sorted.slice(0, limit), + truncated: sorted.length > limit + }); +} +function countParseNode(budget, label) { + budget.nodes += 1; + if (budget.nodes > MAX_PERCOLATION_RESULT_NODES) { + throw new RangeError(`${label} exceeds the ${MAX_PERCOLATION_RESULT_NODES.toLocaleString("en-US")}-node percolation result limit.`); + } +} +function dataRecord(value, label, budget) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`${label} must be a plain data object.`); + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${label} must be a plain data object.`); + } + countParseNode(budget, label); + const output = Object.create(null); + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const key of Reflect.ownKeys(descriptors)) { + if (typeof key !== "string") { + throw new TypeError(`${label} must not contain symbol fields.`); + } + const descriptor = descriptors[key]; + if (descriptor === undefined || !("value" in descriptor) || !descriptor.enumerable) { + throw new TypeError(`${label}.${key} must be an enumerable data property.`); + } + Object.defineProperty(output, key, { + configurable: false, + enumerable: true, + value: descriptor.value, + writable: false + }); + } + return Object.freeze(output); +} +function exactKeys(record, keys, label) { + const actual = Reflect.ownKeys(record); + const expected = new Set(keys); + if (actual.length !== keys.length || actual.some((key) => typeof key !== "string" || !expected.has(key))) { + throw new TypeError(`${label} must contain exactly: ${keys.join(", ")}.`); + } +} +function dataArray(value, label, maximum, budget) { + if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) { + throw new TypeError(`${label} must be an ordinary array.`); + } + if (value.length > maximum) { + throw new RangeError(`${label} exceeds its ${maximum.toLocaleString("en-US")}-entry limit.`); + } + countParseNode(budget, label); + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const key of Reflect.ownKeys(descriptors)) { + if (typeof key !== "string") { + throw new TypeError(`${label} must not contain symbol fields.`); + } + if (key === "length") + continue; + const index = Number(key); + if (!Number.isSafeInteger(index) || index < 0 || index >= value.length || String(index) !== key) { + throw new TypeError(`${label} contains a non-index property.`); + } + } + const output = []; + for (let index = 0;index < value.length; index += 1) { + const descriptor = descriptors[String(index)]; + if (descriptor === undefined || !("value" in descriptor) || !descriptor.enumerable) { + throw new TypeError(`${label} must be a dense array of data properties.`); + } + output.push(descriptor.value); + } + return Object.freeze(output); +} +function hasUnpairedSurrogate(value) { + for (let index = 0;index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 55296 && code <= 56319) { + const next = value.charCodeAt(index + 1); + if (next < 56320 || next > 57343) + return true; + index += 1; + } else if (code >= 56320 && code <= 57343) { + return true; + } + } + return false; +} +function parsedText(value, label, budget, options = {}) { + if (typeof value !== "string" || options.empty !== true && value === "" || hasUnpairedSurrogate(value)) { + throw new TypeError(`${label} must be a bounded Unicode string.`); + } + const bytes = new TextEncoder().encode(value).byteLength; + if (bytes > MAX_PERCOLATION_TEXT_UTF8_BYTES) { + throw new RangeError(`${label} exceeds its ${MAX_PERCOLATION_TEXT_UTF8_BYTES.toLocaleString("en-US")}-byte limit.`); + } + budget.utf8Bytes += bytes; + if (budget.utf8Bytes > MAX_PERCOLATION_RESULT_UTF8_BYTES) { + throw new RangeError(`Percolation result text exceeds ${MAX_PERCOLATION_RESULT_UTF8_BYTES.toLocaleString("en-US")} UTF-8 bytes.`); + } + return value; +} +function canonicalNote(value, label, budget) { + const parsed = parsedText(value, label, budget); + if (!isCanonicalNoteId(parsed)) { + throw new TypeError(`${label} must be a canonical note ID.`); + } + return parsed; +} +function canonicalMarkdownPath(value, label, budget) { + const parsed = parsedText(value, label, budget); + if (!parsed.endsWith(".md") || !isCanonicalNoteId(parsed.slice(0, -3))) { + throw new TypeError(`${label} must be a canonical vault Markdown path.`); + } + return parsed; +} +function canonicalPredicate(value, label, budget) { + const parsed = parsedText(value, label, budget); + if (!isCanonicalRelationPredicate(parsed)) { + throw new TypeError(`${label} must be a canonical relation predicate.`); + } + return parsed; +} +function nullableText(value, label, budget) { + return value === null ? null : parsedText(value, label, budget, { empty: true }); +} +function parsedBoolean(value, label) { + if (typeof value !== "boolean") + throw new TypeError(`${label} must be a boolean.`); + return value; +} +function positiveSafeInteger(value, label, maximum = MAX_PERCOLATION_EVIDENCE) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum) { + throw new TypeError(`${label} must be a positive bounded safe integer.`); + } + return value; +} +function parsedMinSupport(value, label) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < DEFAULT_PERCOLATION_MIN_SUPPORT || value > MAX_PERCOLATION_LIMIT) { + throw new TypeError(`${label} must be an integer from ${DEFAULT_PERCOLATION_MIN_SUPPORT} through ${MAX_PERCOLATION_LIMIT}.`); + } + return value; +} +function predicateDisposition(value, label, budget) { + const record = dataRecord(value, label, budget); + const kind = parsedText(record.kind, `${label}.kind`, budget); + if (kind === "required") { + exactKeys(record, ["kind"], label); + return Object.freeze({ kind: "required" }); + } + if (kind === "suggested") { + exactKeys(record, ["kind", "value"], label); + return Object.freeze({ + kind: "suggested", + value: canonicalPredicate(record.value, `${label}.value`, budget) + }); + } + throw new TypeError(`${label}.kind must be required or suggested.`); +} +function parsedMissingConceptEvidence(value, label, budget) { + const record = dataRecord(value, label, budget); + exactKeys(record, ["kind", "note", "path", "tag"], label); + if (parsedText(record.kind, `${label}.kind`, budget) !== "tag") { + throw new TypeError(`${label}.kind must be tag.`); + } + const note = canonicalNote(record.note, `${label}.note`, budget); + const path = canonicalMarkdownPath(record.path, `${label}.path`, budget); + if (path !== `${note}.md`) + throw new TypeError(`${label}.path must identify its note.`); + return Object.freeze({ + kind: "tag", + note, + path, + tag: parsedText(record.tag, `${label}.tag`, budget) + }); +} +function parsedSharedEvidence(value, label, budget) { + const record = dataRecord(value, label, budget); + const kind = parsedText(record.kind, `${label}.kind`, budget); + if (kind === "shared-tag") { + exactKeys(record, ["kind", "note", "path", "tag"], label); + const note = canonicalNote(record.note, `${label}.note`, budget); + const path = canonicalMarkdownPath(record.path, `${label}.path`, budget); + if (path !== `${note}.md`) + throw new TypeError(`${label}.path must identify its note.`); + return Object.freeze({ + kind: "shared-tag", + note, + path, + tag: parsedText(record.tag, `${label}.tag`, budget) + }); + } + if (kind === "shared-concept") { + exactKeys(record, ["kind", "note", "path", "concept", "conceptPath"], label); + const note = canonicalNote(record.note, `${label}.note`, budget); + const path = canonicalMarkdownPath(record.path, `${label}.path`, budget); + const concept = canonicalNote(record.concept, `${label}.concept`, budget); + const conceptPath = canonicalMarkdownPath(record.conceptPath, `${label}.conceptPath`, budget); + if (path !== `${note}.md`) + throw new TypeError(`${label}.path must identify its note.`); + if (conceptPath !== `${concept}.md`) { + throw new TypeError(`${label}.conceptPath must identify its concept.`); + } + return Object.freeze({ + kind: "shared-concept", + note, + path, + concept, + conceptPath + }); + } + throw new TypeError(`${label}.kind must be shared-tag or shared-concept.`); +} +function parsedMentionEvidence(value, label, budget) { + const record = dataRecord(value, label, budget); + exactKeys(record, ["kind", "source", "target", "line", "phrase"], label); + if (parsedText(record.kind, `${label}.kind`, budget) !== "mention") { + throw new TypeError(`${label}.kind must be mention.`); + } + return Object.freeze({ + kind: "mention", + source: canonicalNote(record.source, `${label}.source`, budget), + target: canonicalNote(record.target, `${label}.target`, budget), + line: positiveSafeInteger(record.line, `${label}.line`), + phrase: parsedText(record.phrase, `${label}.phrase`, budget) + }); +} +function parsedRelationEvidence(value, label, budget) { + const record = dataRecord(value, label, budget); + exactKeys(record, ["kind", "source", "target", "predicate", "line", "authoredTarget"], label); + if (parsedText(record.kind, `${label}.kind`, budget) !== "relation") { + throw new TypeError(`${label}.kind must be relation.`); + } + return Object.freeze({ + kind: "relation", + source: canonicalNote(record.source, `${label}.source`, budget), + target: canonicalNote(record.target, `${label}.target`, budget), + predicate: canonicalPredicate(record.predicate, `${label}.predicate`, budget), + line: positiveSafeInteger(record.line, `${label}.line`), + authoredTarget: parsedText(record.authoredTarget, `${label}.authoredTarget`, budget) + }); +} +function parsedRelationIssueEvidence(value, label, budget) { + const record = dataRecord(value, label, budget); + exactKeys(record, [ + "kind", + "issue", + "source", + "line", + "predicate", + "target", + "candidates", + "candidatesTruncated", + "message" + ], label); + if (parsedText(record.kind, `${label}.kind`, budget) !== "relation-issue") { + throw new TypeError(`${label}.kind must be relation-issue.`); + } + if (record.issue !== "malformed" && record.issue !== "broken" && record.issue !== "ambiguous") { + throw new TypeError(`${label}.issue is unsupported.`); + } + const issue = parsedText(record.issue, `${label}.issue`, budget); + const candidates = dataArray(record.candidates, `${label}.candidates`, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, budget).map((candidate, index) => canonicalNote(candidate, `${label}.candidates[${index}]`, budget)); + for (let index = 0;index < candidates.length; index += 1) { + const previous = candidates[index - 1]; + const candidate = candidates[index]; + if (candidate === undefined) + continue; + if (previous !== undefined && compareText(previous, candidate) >= 0) { + throw new TypeError(`${label}.candidates must be sorted and unique.`); + } + } + if (issue !== "ambiguous" && candidates.length !== 0) { + throw new TypeError(`${label}.candidates are only valid for ambiguous issues.`); + } + if (issue === "ambiguous" && candidates.length < 2) { + throw new TypeError(`${label}.candidates must identify at least two ambiguous notes.`); + } + const candidatesTruncated = parsedBoolean(record.candidatesTruncated, `${label}.candidatesTruncated`); + if (issue !== "ambiguous" && candidatesTruncated || candidatesTruncated && candidates.length !== MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE) { + throw new TypeError(`${label}.candidatesTruncated is inconsistent.`); + } + const predicate = issue === "malformed" ? nullableText(record.predicate, `${label}.predicate`, budget) : canonicalPredicate(record.predicate, `${label}.predicate`, budget); + const target = issue === "malformed" ? nullableText(record.target, `${label}.target`, budget) : canonicalNote(record.target, `${label}.target`, budget); + return Object.freeze({ + kind: "relation-issue", + issue, + source: canonicalNote(record.source, `${label}.source`, budget), + line: positiveSafeInteger(record.line, `${label}.line`), + predicate, + target, + candidates: Object.freeze(candidates), + candidatesTruncated, + message: parsedText(record.message, `${label}.message`, budget) + }); +} +function evidenceArray(value, label, budget, parse) { + const input = dataArray(value, label, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, budget); + if (input.length === 0) + throw new TypeError(`${label} must not be empty.`); + const output = input.map((entry, index) => parse(entry, `${label}[${index}]`, budget)); + const identities = new Set; + for (const entry of output) { + const identity = JSON.stringify(entry); + if (identities.has(identity)) + throw new TypeError(`${label} must be unique.`); + identities.add(identity); + } + return Object.freeze(output); +} +function parsedRelationProblem(value, label, budget) { + const parsed = parsedText(value, label, budget); + if (parsed !== "self-relation" && parsed !== "reciprocal-relation" && parsed !== "malformed-relation" && parsed !== "broken-relation" && parsed !== "ambiguous-relation") + throw new TypeError(`${label} is unsupported.`); + return parsed; +} +function parsedCommonCandidate(record, label) { + return { + support: positiveSafeInteger(record.support, `${label}.support`), + evidenceTruncated: parsedBoolean(record.evidenceTruncated, `${label}.evidenceTruncated`) + }; +} +function parseCandidate(value, label, budget, version) { + const record = dataRecord(value, label, budget); + const kind = parsedText(record.kind, `${label}.kind`, budget); + if (kind === "missing-concept") { + exactKeys(record, [ + "kind", + "tag", + "suggestedId", + "collidesWith", + "support", + "evidenceTruncated", + "evidence" + ], label); + const common = parsedCommonCandidate(record, label); + const tag = parsedText(record.tag, `${label}.tag`, budget); + const evidence = evidenceArray(record.evidence, `${label}.evidence`, budget, parsedMissingConceptEvidence); + if (evidence.some((entry) => entry.tag !== tag)) { + throw new TypeError(`${label}.evidence must support the candidate tag.`); + } + if (!common.evidenceTruncated && common.support !== evidence.length || common.evidenceTruncated && common.support <= evidence.length) { + throw new TypeError(`${label}.support does not match its bounded evidence.`); + } + return Object.freeze({ + kind: "missing-concept", + tag, + suggestedId: canonicalNote(record.suggestedId, `${label}.suggestedId`, budget), + collidesWith: record.collidesWith === null ? null : canonicalNote(record.collidesWith, `${label}.collidesWith`, budget), + ...common, + evidence + }); + } + if (kind === "missing-relation") { + exactKeys(record, version === 1 ? [ + "kind", + "source", + "target", + "suggestedPredicate", + "support", + "evidenceTruncated", + "evidence" + ] : [ + "kind", + "source", + "target", + "predicate", + "support", + "evidenceTruncated", + "evidence" + ], label); + const source = canonicalNote(record.source, `${label}.source`, budget); + const target = canonicalNote(record.target, `${label}.target`, budget); + if (compareText(source, target) >= 0) { + throw new TypeError(`${label} endpoints must be an ordered, distinct pair.`); + } + const common = parsedCommonCandidate(record, label); + const evidence = evidenceArray(record.evidence, `${label}.evidence`, budget, parsedSharedEvidence); + if (evidence.some((entry) => entry.note !== source && entry.note !== target)) { + throw new TypeError(`${label}.evidence must belong to one of the unordered endpoints.`); + } + const signalEndpoints = new Map; + for (const entry of evidence) { + const signal = entry.kind === "shared-tag" ? `tag\x00${entry.tag}` : `concept\x00${entry.concept}`; + const endpoints = signalEndpoints.get(signal) ?? new Set; + endpoints.add(entry.note); + signalEndpoints.set(signal, endpoints); + } + if ([...signalEndpoints.values()].some((endpoints) => endpoints.size !== 2 || !endpoints.has(source) || !endpoints.has(target))) { + throw new TypeError(`${label}.evidence must pair both unordered endpoints per signal.`); + } + if (!common.evidenceTruncated && common.support !== signalEndpoints.size || common.evidenceTruncated && (evidence.length !== MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE || common.support <= signalEndpoints.size)) { + throw new TypeError(`${label}.support does not match its bounded shared signals.`); + } + if (version === 1) { + if (parsedText(record.suggestedPredicate, `${label}.suggestedPredicate`, budget) !== "related-to") { + throw new TypeError(`${label}.suggestedPredicate must be related-to.`); + } + return Object.freeze({ + kind: "missing-relation", + source, + target, + suggestedPredicate: "related-to", + ...common, + evidence + }); + } + return Object.freeze({ + kind: "missing-relation", + source, + target, + predicate: predicateDisposition(record.predicate, `${label}.predicate`, budget), + ...common, + evidence + }); + } + if (kind === "unlinked-mention") { + exactKeys(record, ["kind", "source", "target", "support", "evidenceTruncated", "evidence"], label); + const source = canonicalNote(record.source, `${label}.source`, budget); + const target = canonicalNote(record.target, `${label}.target`, budget); + const common = parsedCommonCandidate(record, label); + const evidence = evidenceArray(record.evidence, `${label}.evidence`, budget, parsedMentionEvidence); + if (evidence.some((entry) => entry.source !== source || entry.target !== target)) { + throw new TypeError(`${label}.evidence must identify the candidate endpoints.`); + } + if (!common.evidenceTruncated && common.support !== evidence.length || common.evidenceTruncated && common.support <= evidence.length) { + throw new TypeError(`${label}.support does not match its bounded evidence.`); + } + return Object.freeze({ + kind: "unlinked-mention", + source, + target, + ...common, + evidence + }); + } + if (kind === "relation-hygiene") { + exactKeys(record, [ + "kind", + "problem", + "source", + "target", + "predicate", + "message", + "support", + "evidenceTruncated", + "evidence" + ], label); + const problem = parsedRelationProblem(record.problem, `${label}.problem`, budget); + const source = canonicalNote(record.source, `${label}.source`, budget); + const target = problem === "malformed-relation" ? nullableText(record.target, `${label}.target`, budget) : record.target === null ? null : canonicalNote(record.target, `${label}.target`, budget); + const predicate = problem === "malformed-relation" ? nullableText(record.predicate, `${label}.predicate`, budget) : record.predicate === null ? null : canonicalPredicate(record.predicate, `${label}.predicate`, budget); + const common = parsedCommonCandidate(record, label); + const relationProblem = problem === "self-relation" || problem === "reciprocal-relation"; + const evidence = relationProblem ? evidenceArray(record.evidence, `${label}.evidence`, budget, parsedRelationEvidence) : evidenceArray(record.evidence, `${label}.evidence`, budget, parsedRelationIssueEvidence); + if (common.support !== evidence.length) { + throw new TypeError(`${label}.support must equal its hygiene evidence count.`); + } + if (relationProblem) { + const relations = evidence; + if (target === null || predicate === null || common.evidenceTruncated || problem === "self-relation" && target !== source || problem === "reciprocal-relation" && (compareText(source, target) >= 0 || relations.length !== 2) || relations.some((entry) => entry.predicate !== predicate || (problem === "self-relation" ? entry.source !== source || entry.target !== target : !(entry.source === source && entry.target === target || entry.source === target && entry.target === source)))) + throw new TypeError(`${label}.evidence must identify the hygiene relation.`); + } else { + const issues = evidence; + const expectedIssue = problem.slice(0, -"-relation".length); + if (issues.some((entry) => entry.source !== source || entry.issue !== expectedIssue || entry.predicate !== predicate || entry.target !== target || entry.message !== record.message) || common.evidenceTruncated !== issues.some((entry) => entry.candidatesTruncated)) { + throw new TypeError(`${label}.evidence must identify the hygiene issue.`); + } + } + return Object.freeze({ + kind: "relation-hygiene", + problem, + source, + target, + predicate, + message: parsedText(record.message, `${label}.message`, budget), + ...common, + evidence + }); + } + throw new TypeError(`${label}.kind is unsupported.`); +} +function parsedCandidates(value, label, budget, version) { + const input = dataArray(value, label, MAX_PERCOLATION_LIMIT, budget); + const output = input.map((entry, index) => parseCandidate(entry, `${label}[${index}]`, budget, version)); + const identities = new Set; + for (let index = 0;index < output.length; index += 1) { + const candidate = output[index]; + if (candidate === undefined) + continue; + const identity = `${candidate.kind}\x00${candidateIdentity(candidate)}`; + if (identities.has(identity)) + throw new TypeError(`${label} must be unique.`); + identities.add(identity); + const previous = output[index - 1]; + if (previous !== undefined && compareCandidates(previous, candidate) > 0) { + throw new TypeError(`${label} must use canonical percolation ordering.`); + } + } + return Object.freeze(output); +} +function parseResultFields(record, label, budget, version) { + return { + candidates: parsedCandidates(record.candidates, `${label}.candidates`, budget, version), + truncated: parsedBoolean(record.truncated, `${label}.truncated`) + }; +} +function parsePercolationResultV1(value) { + const budget = { nodes: 0, utf8Bytes: 0 }; + const record = dataRecord(value, "percolation result v1", budget); + exactKeys(record, ["candidates", "truncated"], "percolation result v1"); + const fields = parseResultFields(record, "percolation result v1", budget, 1); + return Object.freeze({ + candidates: fields.candidates, + truncated: fields.truncated + }); +} +function parsePercolationResultV2(value) { + const budget = { nodes: 0, utf8Bytes: 0 }; + const record = dataRecord(value, "percolation result v2", budget); + exactKeys(record, ["schemaVersion", "candidates", "truncated"], "percolation result v2"); + if (record.schemaVersion !== PERCOLATION_RESULT_SCHEMA_VERSION) { + throw new TypeError("percolation result v2.schemaVersion must be 2."); + } + const fields = parseResultFields(record, "percolation result v2", budget, 2); + return Object.freeze({ + schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION, + candidates: fields.candidates, + truncated: fields.truncated + }); +} +var parsePercolationResult = parsePercolationResultV2; +function parsePercolationCliOutputV1(value) { + const budget = { nodes: 0, utf8Bytes: 0 }; + const label = "percolation CLI output v1"; + const record = dataRecord(value, label, budget); + exactKeys(record, ["root", "note", "minSupport", "candidates", "truncated"], label); + const fields = parseResultFields(record, label, budget, 1); + return Object.freeze({ + root: parsedText(record.root, `${label}.root`, budget), + note: nullableText(record.note, `${label}.note`, budget), + minSupport: parsedMinSupport(record.minSupport, `${label}.minSupport`), + candidates: fields.candidates, + truncated: fields.truncated + }); +} +function parsePercolationCliOutputV2(value) { + const budget = { nodes: 0, utf8Bytes: 0 }; + const label = "percolation CLI output v2"; + const record = dataRecord(value, label, budget); + exactKeys(record, [ + "root", + "note", + "minSupport", + "limit", + "schemaVersion", + "candidates", + "truncated" + ], label); + if (record.schemaVersion !== PERCOLATION_RESULT_SCHEMA_VERSION) { + throw new TypeError(`${label}.schemaVersion must be 2.`); + } + const fields = parseResultFields(record, label, budget, 2); + const limit = positiveSafeInteger(record.limit, `${label}.limit`, MAX_PERCOLATION_LIMIT); + if (fields.candidates.length > limit || fields.truncated && fields.candidates.length !== limit) { + throw new TypeError(`${label}.limit is inconsistent with its candidates.`); + } + return Object.freeze({ + root: parsedText(record.root, `${label}.root`, budget), + note: nullableText(record.note, `${label}.note`, budget), + minSupport: parsedMinSupport(record.minSupport, `${label}.minSupport`), + limit, + schemaVersion: PERCOLATION_RESULT_SCHEMA_VERSION, + candidates: fields.candidates, + truncated: fields.truncated + }); +} +var parsePercolationCliOutput = parsePercolationCliOutputV2; + +export { DEFAULT_PERCOLATION_LIMIT, MAX_PERCOLATION_LIMIT, DEFAULT_PERCOLATION_MIN_SUPPORT, MAX_PERCOLATION_EVIDENCE_PER_CANDIDATE, MAX_PERCOLATION_NOTES, MAX_PERCOLATION_MENTION_PAIRS, MAX_PERCOLATION_MENTIONS, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, PERCOLATION_RESULT_SCHEMA_VERSION, MAX_PERCOLATION_RESULT_NODES, MAX_PERCOLATION_RESULT_UTF8_BYTES, MAX_PERCOLATION_TEXT_UTF8_BYTES, percolateVault, parsePercolationResultV1, parsePercolationResultV2, parsePercolationResult, parsePercolationCliOutputV1, parsePercolationCliOutputV2, parsePercolationCliOutput }; diff --git a/dist/index-cv6fh7z5.js b/dist/index-gm9t95d9.js similarity index 99% rename from dist/index-cv6fh7z5.js rename to dist/index-gm9t95d9.js index 3752b17..8a41fab 100644 --- a/dist/index-cv6fh7z5.js +++ b/dist/index-gm9t95d9.js @@ -7,7 +7,7 @@ import { } from "./index-48pz4jpc.js"; import { lookupNote -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; // src/search.ts var MAX_EXACT_RESULTS = 500; diff --git a/dist/index-01jj6rbv.js b/dist/index-gxr0fctd.js similarity index 99% rename from dist/index-01jj6rbv.js rename to dist/index-gxr0fctd.js index 2da571b..eafb1ab 100644 --- a/dist/index-01jj6rbv.js +++ b/dist/index-gxr0fctd.js @@ -4,9 +4,10 @@ import { } from "./index-3rm7cz6h.js"; import { isCanonicalNoteId, + isCanonicalRelationPredicate, parseDocumentId, parseQualifiedDocumentUri -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; // src/authoring.ts import { createHash, randomUUID } from "crypto"; @@ -40,7 +41,6 @@ import { } from "yaml"; var MAX_NOTE_BYTES = 16 * 1024 * 1024; var NOTE_REVISION_PATTERN = /^sha256:[0-9a-f]{64}$/u; -var PREDICATE_PATTERN = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u; var MAX_PARENT_DIRECTORY_ENTRIES = 1e5; var MAX_RECOVERY_LOCATIONS_PER_NOTE = 8; @@ -113,7 +113,7 @@ function canonicalRelationTarget(value) { } function normalizeRelationPredicate(value) { const normalized = value.trim().normalize("NFC").toLocaleLowerCase("en-US").replaceAll("_", "-").replace(/\s+/gu, "-").replace(/-{2,}/gu, "-"); - if (!PREDICATE_PATTERN.test(normalized)) { + if (!isCanonicalRelationPredicate(normalized)) { throw new TypeError(`not a valid relation predicate: ${JSON.stringify(value)}`); } return normalized; diff --git a/dist/index-s2gw5aw9.js b/dist/index-qwgsmtsz.js similarity index 99% rename from dist/index-s2gw5aw9.js rename to dist/index-qwgsmtsz.js index 28da10f..c59602c 100644 --- a/dist/index-s2gw5aw9.js +++ b/dist/index-qwgsmtsz.js @@ -1,7 +1,7 @@ // @bun import { fuseRankedCandidates -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; // src/benchmark.ts var MAX_BENCHMARK_CASES = 500; diff --git a/dist/index-zzhgcwyt.js b/dist/index-vxmf14m1.js similarity index 99% rename from dist/index-zzhgcwyt.js rename to dist/index-vxmf14m1.js index 6a87d24..4c7ff81 100644 --- a/dist/index-zzhgcwyt.js +++ b/dist/index-vxmf14m1.js @@ -8,7 +8,7 @@ import { openSemanticSearchSession, recommendedEmbeddingModel, scanVault -} from "./index-zxdy5pby.js"; +} from "./index-5m2ydj5q.js"; import { UntrustedContentBudgetError, createUntrustedToolResult @@ -27,7 +27,7 @@ import { fuseRankedCandidates, searchExactVault, validateSearchQuery -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; import { NavigationBudgetError, navigateLinks @@ -37,7 +37,7 @@ import { } from "./index-48pz4jpc.js"; import { lookupNote -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; // src/sdk.ts import { resolve } from "path"; diff --git a/dist/index-n5dd7r0v.js b/dist/index-xw9ac71d.js similarity index 99% rename from dist/index-n5dd7r0v.js rename to dist/index-xw9ac71d.js index 2438f72..60403ce 100644 --- a/dist/index-n5dd7r0v.js +++ b/dist/index-xw9ac71d.js @@ -2,10 +2,10 @@ import { MAX_SEARCH_RESULTS, openKnowledgeBase -} from "./index-zzhgcwyt.js"; +} from "./index-vxmf14m1.js"; import { scanVault -} from "./index-zxdy5pby.js"; +} from "./index-5m2ydj5q.js"; import { runGitCommand } from "./index-1gwbassd.js"; diff --git a/dist/index-1vrd1rmn.js b/dist/index-ykvvkd77.js similarity index 97% rename from dist/index-1vrd1rmn.js rename to dist/index-ykvvkd77.js index b4dc0c9..6b1a823 100644 --- a/dist/index-1vrd1rmn.js +++ b/dist/index-ykvvkd77.js @@ -4,7 +4,7 @@ import { } from "./index-3v2z4f0q.js"; import { packUntrustedSearchContext -} from "./index-zzhgcwyt.js"; +} from "./index-vxmf14m1.js"; // src/workflows/decision-context.ts var decisionContextWorkflow = defineWorkflow({ diff --git a/dist/index.js b/dist/index.js index 0a73e51..27a94c3 100644 --- a/dist/index.js +++ b/dist/index.js @@ -29,15 +29,25 @@ import { MAX_PERCOLATION_MENTIONS, MAX_PERCOLATION_MENTION_PAIRS, MAX_PERCOLATION_NOTES, + MAX_PERCOLATION_RESULT_NODES, + MAX_PERCOLATION_RESULT_UTF8_BYTES, + MAX_PERCOLATION_TEXT_UTF8_BYTES, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, + PERCOLATION_RESULT_SCHEMA_VERSION, + parsePercolationCliOutput, + parsePercolationCliOutputV1, + parsePercolationCliOutputV2, + parsePercolationResult, + parsePercolationResultV1, + parsePercolationResultV2, percolateVault -} from "./index-dyqwejk5.js"; +} from "./index-f9fy4w1n.js"; import { FrozenEvaluationSnapshotError, knowledgeBaseEvaluationRetrieverIds, openKnowledgeBaseEvaluation, verifyFrozenEvaluationSnapshot -} from "./index-n5dd7r0v.js"; +} from "./index-xw9ac71d.js"; import { DEFAULT_SEARCH_RESULTS, MAX_SEARCH_CANDIDATES, @@ -49,7 +59,7 @@ import { packSearchContext, packUntrustedSearchContext, validateKnowledgeBaseSearchHistory -} from "./index-zzhgcwyt.js"; +} from "./index-vxmf14m1.js"; import"./index-adx6khj5.js"; import { MAX_EMBEDDING_MODEL_BYTES, @@ -75,7 +85,7 @@ import { searchSemanticVault, semanticDatabasePath, sha256EmbeddingModelFile -} from "./index-zxdy5pby.js"; +} from "./index-5m2ydj5q.js"; import"./index-4j3tt0c3.js"; import { GitHistoryError, @@ -144,14 +154,14 @@ import { normalizeRelationPredicate, noteRevision, removeNoteRelation -} from "./index-01jj6rbv.js"; +} from "./index-gxr0fctd.js"; import"./index-3rm7cz6h.js"; import { createRepresentativeRetrievalFixture, createSyntheticRankFusionFixture, evaluateRanking, evaluateRetrievalBenchmark -} from "./index-s2gw5aw9.js"; +} from "./index-qwgsmtsz.js"; import { MAX_SEARCH_QUERY_BYTES, MAX_SEARCH_QUERY_TERMS, @@ -159,7 +169,7 @@ import { fuseRankedCandidates, searchExactVault, validateSearchQuery -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; import { MAX_NAVIGATION_INDEXED_CONNECTIONS, MAX_NAVIGATION_RETURNED_CONNECTIONS, @@ -235,6 +245,7 @@ import { catalogEnd, catalogStart, isCanonicalNoteId, + isCanonicalRelationPredicate, lookupNote, metadataValueFromUnknown, normalizeVaultPath, @@ -243,7 +254,7 @@ import { replaceCatalog, searchableMarkdown, wikiLinks -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; import"./index-1xxnjn0d.js"; // src/oh-adoption.ts import { createHash } from "crypto"; @@ -593,6 +604,12 @@ export { planStatuses, percolateVault, parseRetrievalEvaluationCorpus, + parsePercolationResultV2, + parsePercolationResultV1, + parsePercolationResult, + parsePercolationCliOutputV2, + parsePercolationCliOutputV1, + parsePercolationCliOutput, parseNote, parseLocalAttachmentReferences, parseGitHistoryOutput, @@ -618,6 +635,7 @@ export { knowledgeBaseEvaluationRetrieverIds, isTerminalPlanStatus, isPlanStatus, + isCanonicalRelationPredicate, isCanonicalNoteId, isActivePlanStatus, inspectRepositoryScopeState, @@ -677,6 +695,7 @@ export { RepositoryScopeError, RETRIEVAL_EVALUATION_SCHEMA_VERSION, RETRIEVAL_EVALUATION_REPORT_VERSION, + PERCOLATION_RESULT_SCHEMA_VERSION, NoteRevisionConflictError, NoteRecoveryRequiredError, NoteAlreadyExistsError, @@ -714,6 +733,9 @@ export { MAX_QUERY_METADATA_PATH_SEGMENTS, MAX_QUERY_FILTER_VALUES, MAX_QUERY_FILTERS, + MAX_PERCOLATION_TEXT_UTF8_BYTES, + MAX_PERCOLATION_RESULT_UTF8_BYTES, + MAX_PERCOLATION_RESULT_NODES, MAX_PERCOLATION_NOTES, MAX_PERCOLATION_MENTION_PAIRS, MAX_PERCOLATION_MENTIONS, diff --git a/dist/percolate.js b/dist/percolate.js index b3246ef..9715f4d 100644 --- a/dist/percolate.js +++ b/dist/percolate.js @@ -7,13 +7,33 @@ import { MAX_PERCOLATION_MENTIONS, MAX_PERCOLATION_MENTION_PAIRS, MAX_PERCOLATION_NOTES, + MAX_PERCOLATION_RESULT_NODES, + MAX_PERCOLATION_RESULT_UTF8_BYTES, + MAX_PERCOLATION_TEXT_UTF8_BYTES, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, + PERCOLATION_RESULT_SCHEMA_VERSION, + parsePercolationCliOutput, + parsePercolationCliOutputV1, + parsePercolationCliOutputV2, + parsePercolationResult, + parsePercolationResultV1, + parsePercolationResultV2, percolateVault -} from "./index-dyqwejk5.js"; -import"./index-cxfrakt7.js"; +} from "./index-f9fy4w1n.js"; +import"./index-ekpwvbra.js"; export { percolateVault, + parsePercolationResultV2, + parsePercolationResultV1, + parsePercolationResult, + parsePercolationCliOutputV2, + parsePercolationCliOutputV1, + parsePercolationCliOutput, + PERCOLATION_RESULT_SCHEMA_VERSION, MAX_SCOPED_PERCOLATION_MENTION_PAIRS, + MAX_PERCOLATION_TEXT_UTF8_BYTES, + MAX_PERCOLATION_RESULT_UTF8_BYTES, + MAX_PERCOLATION_RESULT_NODES, MAX_PERCOLATION_NOTES, MAX_PERCOLATION_MENTION_PAIRS, MAX_PERCOLATION_MENTIONS, diff --git a/dist/portfolio.js b/dist/portfolio.js index 81f467a..af362ae 100644 --- a/dist/portfolio.js +++ b/dist/portfolio.js @@ -20,14 +20,14 @@ import { selectAuthorizedVaults, snapshotPortfolioRegistry, validateResolvedPortfolioVaults -} from "./index-jsmvyyvf.js"; -import"./index-zzhgcwyt.js"; +} from "./index-ey46z1zf.js"; +import"./index-vxmf14m1.js"; import"./index-adx6khj5.js"; -import"./index-zxdy5pby.js"; +import"./index-5m2ydj5q.js"; import"./index-4j3tt0c3.js"; import"./index-1gwbassd.js"; import"./index-x3fthpsc.js"; -import"./index-cv6fh7z5.js"; +import"./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; @@ -42,7 +42,7 @@ import { parseVaultKey, portfolioDocumentIdentity, portfolioVaultIdentity -} from "./index-cxfrakt7.js"; +} from "./index-ekpwvbra.js"; import"./index-1xxnjn0d.js"; export { validateResolvedPortfolioVaults, diff --git a/dist/sdk.js b/dist/sdk.js index b5b8c11..895d80e 100644 --- a/dist/sdk.js +++ b/dist/sdk.js @@ -10,17 +10,17 @@ import { packSearchContext, packUntrustedSearchContext, validateKnowledgeBaseSearchHistory -} from "./index-zzhgcwyt.js"; +} from "./index-vxmf14m1.js"; import"./index-adx6khj5.js"; -import"./index-zxdy5pby.js"; +import"./index-5m2ydj5q.js"; import"./index-4j3tt0c3.js"; import"./index-1gwbassd.js"; -import"./index-cv6fh7z5.js"; +import"./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; import"./index-5vwpzb5a.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; import"./index-1xxnjn0d.js"; export { validateKnowledgeBaseSearchHistory, diff --git a/dist/search.js b/dist/search.js index dcffc82..0be3437 100644 --- a/dist/search.js +++ b/dist/search.js @@ -6,12 +6,12 @@ import { fuseRankedCandidates, searchExactVault, validateSearchQuery -} from "./index-cv6fh7z5.js"; +} from "./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; import"./index-5vwpzb5a.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; export { validateSearchQuery, searchExactVault, diff --git a/dist/semantic.js b/dist/semantic.js index 33f306b..a96e4b4 100644 --- a/dist/semantic.js +++ b/dist/semantic.js @@ -14,13 +14,13 @@ import { searchSemanticVault, semanticDatabasePath, sha256EmbeddingModelFile -} from "./index-zxdy5pby.js"; -import"./index-cv6fh7z5.js"; +} from "./index-5m2ydj5q.js"; +import"./index-gm9t95d9.js"; import"./index-d13v9ckt.js"; import"./index-48pz4jpc.js"; import"./index-06c9ctr6.js"; import"./index-5vwpzb5a.js"; -import"./index-cxfrakt7.js"; +import"./index-ekpwvbra.js"; export { sha256EmbeddingModelFile, semanticDatabasePath, diff --git a/dist/workflows/decision-context.js b/dist/workflows/decision-context.js index 4600b53..7d943ea 100644 --- a/dist/workflows/decision-context.js +++ b/dist/workflows/decision-context.js @@ -1,19 +1,19 @@ // @bun import { decisionContextWorkflow -} from "../index-1vrd1rmn.js"; +} from "../index-ykvvkd77.js"; import"../index-3v2z4f0q.js"; -import"../index-zzhgcwyt.js"; +import"../index-vxmf14m1.js"; import"../index-adx6khj5.js"; -import"../index-zxdy5pby.js"; +import"../index-5m2ydj5q.js"; import"../index-4j3tt0c3.js"; import"../index-1gwbassd.js"; -import"../index-cv6fh7z5.js"; +import"../index-gm9t95d9.js"; import"../index-d13v9ckt.js"; import"../index-48pz4jpc.js"; import"../index-06c9ctr6.js"; import"../index-5vwpzb5a.js"; -import"../index-cxfrakt7.js"; +import"../index-ekpwvbra.js"; import"../index-1xxnjn0d.js"; export { decisionContextWorkflow diff --git a/dist/workflows/index.js b/dist/workflows/index.js index 0d2af2f..a7e1793 100644 --- a/dist/workflows/index.js +++ b/dist/workflows/index.js @@ -1,7 +1,7 @@ // @bun import { decisionContextWorkflow -} from "../index-1vrd1rmn.js"; +} from "../index-ykvvkd77.js"; import { explainChangeWorkflow } from "../index-zr53sf63.js"; @@ -9,17 +9,17 @@ import { planRadarWorkflow } from "../index-vvdj6kca.js"; import"../index-3v2z4f0q.js"; -import"../index-zzhgcwyt.js"; +import"../index-vxmf14m1.js"; import"../index-adx6khj5.js"; -import"../index-zxdy5pby.js"; +import"../index-5m2ydj5q.js"; import"../index-4j3tt0c3.js"; import"../index-1gwbassd.js"; -import"../index-cv6fh7z5.js"; +import"../index-gm9t95d9.js"; import"../index-d13v9ckt.js"; import"../index-48pz4jpc.js"; import"../index-06c9ctr6.js"; import"../index-5vwpzb5a.js"; -import"../index-cxfrakt7.js"; +import"../index-ekpwvbra.js"; import"../index-1xxnjn0d.js"; export { planRadarWorkflow, diff --git a/scripts/kb-skill-contract.test.ts b/scripts/kb-skill-contract.test.ts index 1897581..49d5cef 100644 --- a/scripts/kb-skill-contract.test.ts +++ b/scripts/kb-skill-contract.test.ts @@ -15,6 +15,25 @@ import { const root = "/approved/skills"; +async function regularFiles(directory: string, prefix = ""): Promise { + const files: string[] = []; + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) { + const relativePath = prefix === "" ? entry.name : `${prefix}/${entry.name}`; + if (entry.isSymbolicLink()) { + throw new Error(`skill resources must not contain symbolic links: ${relativePath}`); + } + if (entry.isDirectory()) { + files.push(...await regularFiles(resolve(directory, entry.name), relativePath)); + } else if (entry.isFile()) { + files.push(relativePath); + } else { + throw new Error(`skill resources must be regular files or directories: ${relativePath}`); + } + } + return files; +} + function proposal( overrides: Partial = {}, ): CustomizationProposal { @@ -100,7 +119,7 @@ test("the shipped skill resources preserve routing and companion contracts", asy readFile(resolve(repositoryRoot, "src/cli.ts"), "utf8"), readFile(resolve(repositoryRoot, "src/index.ts"), "utf8"), readFile(resolve(repositoryRoot, "package.json"), "utf8"), - readdir(resolve(repositoryRoot, "skills/kb"), { recursive: true }), + regularFiles(resolve(repositoryRoot, "skills/kb")), ]); const manifest = JSON.parse(manifestSource) as { readonly exports?: unknown; diff --git a/scripts/npm-package-identity.ts b/scripts/npm-package-identity.ts index b6e9fb5..10778dd 100644 --- a/scripts/npm-package-identity.ts +++ b/scripts/npm-package-identity.ts @@ -11,6 +11,7 @@ import { const packageName = "@hraness/kb"; const npmRegistry = "https://registry.npmjs.org"; const stableVersionPattern = /^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/u; +const ohAdoptionPreparerIntroduction = [0n, 18n, 0n] as const; type NpmPackFile = Readonly<{ mode: number; path: string; size: number }>; type NpmPackIdentity = Readonly<{ @@ -55,6 +56,27 @@ export type VerifiedNpmPackageIdentity = Readonly<{ version: string; }>; +function stableVersionParts(version: string): readonly [bigint, bigint, bigint] { + const match = stableVersionPattern.exec(version); + if (match === null || match[1] === undefined || match[2] === undefined || match[3] === undefined) { + throw new TypeError(`Package version must be a canonical stable semantic version: ${version}`); + } + return [BigInt(match[1]), BigInt(match[2]), BigInt(match[3])]; +} + +export function requiresOhAdoptionPreparerExport(packageVersion: string): boolean { + const current = stableVersionParts(packageVersion); + for (let index = 0; index < current.length; index += 1) { + const currentPart = current[index]; + const introductionPart = ohAdoptionPreparerIntroduction[index]; + if (currentPart === undefined || introductionPart === undefined) { + throw new TypeError("Stable semantic version comparison is incomplete"); + } + if (currentPart !== introductionPart) return currentPart > introductionPart; + } + return true; +} + function record(value: unknown, label: string): Record { if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new Error(`${label} must be an object`); diff --git a/scripts/npm-release-workflow.test.ts b/scripts/npm-release-workflow.test.ts index fc4fbcd..8c1d2f9 100644 --- a/scripts/npm-release-workflow.test.ts +++ b/scripts/npm-release-workflow.test.ts @@ -10,7 +10,10 @@ import { inspectPackageArtifact, type PackageArtifactInventory, } from "./package-artifact.js"; -import { verifyNpmPackageIdentity } from "./npm-package-identity.js"; +import { + requiresOhAdoptionPreparerExport, + verifyNpmPackageIdentity, +} from "./npm-package-identity.js"; const stageWorkflowUrl = new URL("../.github/workflows/npm-stage.yml", import.meta.url); const releaseWorkflowUrl = new URL("../.github/workflows/release.yml", import.meta.url); @@ -41,6 +44,32 @@ function integrity(bytes: Uint8Array): string { return `sha512-${createHash("sha512").update(bytes).digest("base64")}`; } +describe("package smoke version policy", () => { + test("requires the Oh adoption preparer only from its stable introduction", () => { + expect(requiresOhAdoptionPreparerExport("0.17.1")).toBe(false); + expect(requiresOhAdoptionPreparerExport("0.17.3")).toBe(false); + expect(requiresOhAdoptionPreparerExport("0.18.0")).toBe(true); + expect(requiresOhAdoptionPreparerExport("0.18.1")).toBe(true); + expect(requiresOhAdoptionPreparerExport("1.0.0")).toBe(true); + }); + + test("rejects noncanonical or non-stable package versions", () => { + for (const version of [ + "", + "v0.18.0", + "0.18", + "0.18.0-beta.1", + "00.18.0", + "0.018.0", + "0.18.00", + ]) { + expect(() => requiresOhAdoptionPreparerExport(version)).toThrow( + "canonical stable semantic version", + ); + } + }); +}); + function packJson( bytes: Uint8Array, inventory: PackageArtifactInventory, @@ -409,8 +438,8 @@ describe("canonical npm package identity", () => { sourcePackJson, }); const verified = await verifyNpmPackageIdentity(validInput); - expect(verified.fileCount).toBe(201); - expect(verified.unpackedBytes).toBe(4_895_276); + expect(verified.fileCount).toBe(204); + expect(verified.unpackedBytes).toBe(4_974_382); expect(verified.sourceArchiveSha512).not.toBe(verified.registryArchiveSha512); const originalTar = gunzipSync(sourceBytes); diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index 37b4d12..36b1fae 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -7,6 +7,7 @@ import { inspectPackageArtifact, type PackageArtifactInventory, } from "./package-artifact.js"; +import { requiresOhAdoptionPreparerExport } from "./npm-package-identity.js"; const packageName = "@hraness/kb"; const maximumPackageFiles = 210; @@ -59,8 +60,7 @@ const importSpecifiers = [ "@hraness/kb/workflows/explain-change", "@hraness/kb/workflows/plan-radar", ]; -const requiredNamedExports = { - "@hraness/kb": ["createOhAdoptionPreparerV1"], +const baselineRequiredNamedExports = { "@hraness/kb/clip/bundle-reader": ["readCaptureBundle", "verifyCaptureBundle"], "@hraness/kb/clip/jobs": ["createCaptureJob", "openCaptureJobStore", "updateCaptureJob"], "@hraness/kb/clip/refresh": ["diffCaptureBundle"], @@ -541,6 +541,13 @@ try { if (sourceManifest.name !== packageName || typeof sourceManifest.version !== "string") { throw new Error("source package identity is invalid"); } + const requiresOhAdoptionPreparer = requiresOhAdoptionPreparerExport(sourceManifest.version); + const requiredNamedExports = requiresOhAdoptionPreparer + ? { + "@hraness/kb": ["createOhAdoptionPreparerV1"], + ...baselineRequiredNamedExports, + } + : baselineRequiredNamedExports; const inventory = await inspectPackageArtifact(archive); if (packageInput.packJson !== undefined) { await verifyExactNpmPackMetadata( @@ -626,7 +633,9 @@ for (const specifier of ${JSON.stringify(importSpecifiers)}) { const consumerSource = `${importSpecifiers.map((specifier, index) => `import * as surface${String(index)} from ${JSON.stringify(specifier)};` ).join("\n")} -import { createOhAdoptionPreparerV1 } from "@hraness/kb"; +${requiresOhAdoptionPreparer + ? 'import { createOhAdoptionPreparerV1 } from "@hraness/kb";' + : ""} import { readCaptureBundle, verifyCaptureBundle } from "@hraness/kb/clip/bundle-reader"; import { createCaptureJob, openCaptureJobStore, updateCaptureJob } from "@hraness/kb/clip/jobs"; import { diffCaptureBundle } from "@hraness/kb/clip/refresh"; @@ -646,7 +655,7 @@ const registry = parsePortfolioRegistry({ const identity = parseQualifiedDocumentUri("kb://hraness/kb/note-id"); const projected = projectUntrustedJson([{ title: "stored source" }]); void [ - createOhAdoptionPreparerV1, + ${requiresOhAdoptionPreparer ? "createOhAdoptionPreparerV1," : ""} readCaptureBundle, verifyCaptureBundle, createCaptureJob, openCaptureJobStore, updateCaptureJob, diffCaptureBundle, openKnowledgePortfolio, prioritizeSearchHits,