feat: per-file metadata — PR/issue context, custom tags, and search

.changeset/file-metadata.md

@@ -0,0 +1,13 @@
+---
+"@buildinternet/uploads": minor
+---
+
+Add queryable custom metadata to the CLI: `put --meta k=v` (repeatable), `attach`
+now writes `gh.repo`/`gh.kind`/`gh.number`/`gh.ref` automatically (plus its own
+`--meta` extras), new `meta get`/`meta set` commands, `list --meta k=v` and the
+`find k=v...` alias for filtering objects by metadata.
+
+MCP parity: the local stdio MCP's `put`/`attach` tools gain a `metadata` param
+(same gh.\* auto-injection as `attach`), and two new tools — `set_metadata`
+(merge-set/delete) and `find_files` (metadata filter) — mirror the CLI's
+`meta set`/`find`. The hosted MCP's `put` tool also gains a `metadata` param.

apps/api/migrations/20260713210559_file_metadata.sql

@@ -0,0 +1,10 @@
+CREATE TABLE file_metadata (
+  workspace  TEXT NOT NULL,
+  object_key TEXT NOT NULL,
+  meta_key   TEXT NOT NULL,
+  meta_value TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY (workspace, object_key, meta_key)
+);
+CREATE INDEX file_metadata_lookup_idx
+  ON file_metadata (workspace, meta_key, meta_value);

apps/api/package.json

@@ -8,6 +8,7 @@
     "./workspace": "./src/workspace.ts",
     "./storage": "./src/storage.ts",
     "./files": "./src/files-core.ts",
+    "./file-metadata": "./src/file-metadata.ts",
     "./guards": "./src/guards.ts",
     "./budget": "./src/budget.ts",
     "./usage": "./src/usage.ts",

apps/api/scripts/backfill-gh-metadata.d.mts

@@ -0,0 +1,43 @@
+/** Type declarations for backfill-gh-metadata.mjs, consumed by test/backfill-gh-metadata.test.ts. */
+
+export const GH_KEY_RE: RegExp;
+
+export interface GhMetadataPlan {
+  key: string;
+  metadata: {
+    "gh.repo": string;
+    "gh.kind": "pull" | "issue";
+    "gh.number": string;
+    "gh.ref": string;
+  };
+}
+
+export function planForKey(key: string): GhMetadataPlan | null;
+
+export interface RunBackfillSummary {
+  matched: number;
+  patched: number;
+  skipped: number;
+  errors: number;
+}
+
+/** Minimal fetch-Response shape the loop consumes — real `fetch` and test stubs both satisfy it. */
+export interface FetchLikeResponse {
+  ok: boolean;
+  status: number;
+  json(): Promise<unknown>;
+}
+
+export type FetchLike = (url: string, init?: Record<string, unknown>) => Promise<FetchLikeResponse>;
+
+export interface RunBackfillOptions {
+  apiUrl: string;
+  workspace: string;
+  token: string;
+  dryRun?: boolean;
+  prefix?: string;
+  fetchImpl?: FetchLike;
+  log?: (message: string) => void;
+}
+
+export function runBackfill(options: RunBackfillOptions): Promise<RunBackfillSummary>;

apps/api/scripts/backfill-gh-metadata.mjs

@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+/**
+ * One-time backfill: derive gh.* metadata for objects uploaded under gh/...
+ * before per-file metadata existed, so they become searchable the same way
+ * `uploads attach` writes them going forward (see ghMetadataFromTarget in
+ * packages/uploads/src/github.ts — this script mirrors that mapping).
+ *
+ * Pages GET /v1/:ws/files?prefix=gh/ (cursor loop), parses each key against
+ * GH_KEY_RE, and PATCHes /v1/:ws/files/<key>/metadata with `{ set: {...} }`
+ * for every match. Non-matching keys under gh/ are skipped and counted, not
+ * treated as errors. Merge-semantics PATCH means re-running is harmless
+ * (idempotent) — safe to re-run after interruption.
+ *
+ * BINDING: gh.* metadata values are canonical lowercase (gh.repo, gh.ref);
+ * the object key itself keeps its original casing untouched. gh.kind is
+ * "issue" for keys under gh/.../issues/... and "pull" for gh/.../pull/....
+ *
+ * Usage (from apps/api, with UPLOADS_API_URL / UPLOADS_WORKSPACE /
+ * UPLOADS_TOKEN in the monorepo-root .env, same names as .env.example):
+ *   node --env-file=../../.env scripts/backfill-gh-metadata.mjs --dry-run
+ *   node --env-file=../../.env scripts/backfill-gh-metadata.mjs
+ *   node --env-file=../../.env scripts/backfill-gh-metadata.mjs --workspace other-ws
+ *
+ * Never point this at a production workspace during testing — use a local
+ * `wrangler dev` stack (UPLOADS_API_URL=http://localhost:8787) first.
+ */
+import { pathToFileURL } from "node:url";
+
+/** Anchored gh/<owner>/<repo>/<pull|issues>/<number>/ prefix — see task brief. */
+export const GH_KEY_RE = /^gh\/([^/]+)\/([^/]+)\/(pull|issues)\/(\d+)\//;
+
+/**
+ * Pure key-parsing + plan-building logic, no I/O. Returns the PATCH plan for
+ * a matching key, or null when the key doesn't match the gh/ layout (caller
+ * counts those as skipped, not errors).
+ */
+export function planForKey(key) {
+  const m = GH_KEY_RE.exec(key);
+  if (!m) return null;
+  const [, ownerRaw, repoRaw, kindSeg, number] = m;
+  const repo = `${ownerRaw}/${repoRaw}`.toLowerCase();
+  const kind = kindSeg === "issues" ? "issue" : "pull";
+  return {
+    key,
+    metadata: {
+      "gh.repo": repo,
+      "gh.kind": kind,
+      "gh.number": number,
+      "gh.ref": `${repo}#${number}`,
+    },
+  };
+}
+
+/**
+ * Orchestrates the list→plan→PATCH loop. `fetchImpl` is injectable for
+ * tests; production callers pass the global `fetch`.
+ *
+ * @returns {Promise<{matched: number, patched: number, skipped: number, errors: number}>}
+ */
+export async function runBackfill({
+  apiUrl,
+  workspace,
+  token,
+  dryRun = false,
+  prefix = "gh/",
+  fetchImpl = fetch,
+  log = console.log,
+}) {
+  const base = apiUrl.replace(/\/$/, "");
+  const counts = { matched: 0, patched: 0, skipped: 0, errors: 0 };
+  let cursor;
+
+  for (;;) {
+    const listUrl = new URL(`${base}/v1/${workspace}/files`);
+    listUrl.searchParams.set("prefix", prefix);
+    if (cursor) listUrl.searchParams.set("cursor", cursor);
+
+    // A failed list call (non-2xx or transport error) aborts the loop —
+    // without the page we can't discover further keys. A failed PATCH below
+    // only skips that one key. Both count as errors (non-zero exit).
+    let listRes;
+    try {
+      listRes = await fetchImpl(listUrl.toString(), {
+        headers: { Authorization: `Bearer ${token}` },
+      });
+    } catch (err) {
+      counts.errors += 1;
+      log(`error: list failed (${err instanceof Error ? err.message : String(err)})`);
+      break;
+    }
+    if (!listRes.ok) {
+      counts.errors += 1;
+      log(`error: list failed (HTTP ${listRes.status})`);
+      break;
+    }
+    const page = await listRes.json();
+    const items = Array.isArray(page.items) ? page.items : [];
+
+    for (const item of items) {
+      const plan = planForKey(item.key);
+      if (!plan) {
+        counts.skipped += 1;
+        log(`skip  ${item.key} (does not match gh/ layout)`);
+        continue;
+      }
+      counts.matched += 1;
+
+      if (dryRun) {
+        log(`[dry-run] would PATCH ${plan.key} set=${JSON.stringify(plan.metadata)}`);
+        continue;
+      }
+
+      const patchUrl = `${base}/v1/${workspace}/files/${plan.key}/metadata`;
+      let patchRes;
+      try {
+        patchRes = await fetchImpl(patchUrl, {
+          method: "PATCH",
+          headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({ set: plan.metadata }),
+        });
+      } catch (err) {
+        counts.errors += 1;
+        log(
+          `error: PATCH ${plan.key} failed (${err instanceof Error ? err.message : String(err)})`,
+        );
+        continue;
+      }
+      if (!patchRes.ok) {
+        counts.errors += 1;
+        log(`error: PATCH ${plan.key} failed (HTTP ${patchRes.status})`);
+        continue;
+      }
+      counts.patched += 1;
+      log(`patch ${plan.key} set=${JSON.stringify(plan.metadata)}`);
+    }
+
+    cursor = page.cursor;
+    if (!cursor) break;
+  }
+
+  return counts;
+}
+
+function parseArgs(argv) {
+  const opts = { dryRun: false, workspace: undefined };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--dry-run") {
+      opts.dryRun = true;
+    } else if (arg === "--workspace") {
+      opts.workspace = argv[++i];
+    } else {
+      throw new Error(`unexpected argument: ${arg}`);
+    }
+  }
+  return opts;
+}
+
+async function main() {
+  let opts;
+  try {
+    opts = parseArgs(process.argv.slice(2));
+  } catch (err) {
+    console.error(`error: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+  }
+  const apiUrl = process.env.UPLOADS_API_URL ?? "https://api.uploads.sh";
+  const workspace = opts.workspace ?? process.env.UPLOADS_WORKSPACE;
+  const token = process.env.UPLOADS_TOKEN;
+
+  if (!workspace) {
+    console.error("error: UPLOADS_WORKSPACE (or --workspace) is required");
+    process.exit(1);
+  }
+  if (!token) {
+    console.error("error: UPLOADS_TOKEN is required");
+    process.exit(1);
+  }
+
+  console.log(
+    `backfilling gh.* metadata for ${apiUrl} workspace=${workspace}${opts.dryRun ? " (dry-run)" : ""}`,
+  );
+
+  // runBackfill handles fetch failures itself; this catch is the last line of
+  // defense (e.g. a malformed list body) — exit non-zero, no raw stack dump.
+  let summary;
+  try {
+    summary = await runBackfill({ apiUrl, workspace, token, dryRun: opts.dryRun });
+  } catch (err) {
+    console.error(`error: backfill failed (${err instanceof Error ? err.message : String(err)})`);
+    process.exit(1);
+  }
+
+  console.log(
+    `\ndone: matched=${summary.matched} patched=${summary.patched} skipped=${summary.skipped} errors=${summary.errors}`,
+  );
+  process.exit(summary.errors > 0 ? 1 : 0);
+}
+
+// Only run when executed directly (`node backfill-gh-metadata.mjs`), not when
+// imported for its pure functions (test/backfill-gh-metadata.test.ts).
+const isMain = (() => {
+  try {
+    return Boolean(process.argv[1]) && import.meta.url === pathToFileURL(process.argv[1]).href;
+  } catch {
+    return false;
+  }
+})();
+if (isMain) {
+  main();
+}