fix: harden middlecache downloads and consume OpenAPI archive

bin/fetch-openapi.ts

@@ -0,0 +1,44 @@
+#!/usr/bin/env tsx
+
+import fs from "fs";
+
+import {
+	fetchOpenApiSchema,
+	getOpenApiJsonPath,
+} from "../src/util/openapi-schema";
+
+// --soft: warn and continue on failure instead of exiting non-zero.
+//         Used by the predev hook so a network failure doesn't block local development.
+// --force: re-fetch even if the schema already exists.
+const soft = process.argv.includes("--soft");
+const force = process.argv.includes("--force");
+
+const fail = (message: string): never => {
+	if (soft) {
+		console.warn(
+			`Warning: ${message} — API endpoint pages will not work without the schema`,
+		);
+		process.exit(0);
+	}
+	console.error(`Error: ${message}`);
+	process.exit(1);
+};
+
+const openapiFile = getOpenApiJsonPath();
+
+if (fs.existsSync(openapiFile) && !force) {
+	console.log(
+		"OpenAPI schema already exists, skipping fetch. (run `pnpm tsx bin/fetch-openapi.ts --force` to re-fetch)",
+	);
+	process.exit(0);
+}
+
+console.log("Fetching Cloudflare API OpenAPI schema from middlecache");
+
+try {
+	await fetchOpenApiSchema();
+} catch (err) {
+	fail(`fetch failed: ${err}`);
+}
+
+console.log("OpenAPI schema ready");

bin/fetch-skills.ts

@@ -1,10 +1,12 @@
 #!/usr/bin/env tsx
 
-import { spawn } from "child_process";
 import fs from "fs";
 import { join } from "path";
 
-import { downloadToDotTempIfNotPresent } from "../src/util/custom-loaders";
+import {
+	downloadToDotTempIfNotPresent,
+	extractTarGz,
+} from "../src/util/custom-loaders";
 
 const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/";
 const SKILLS_MIDDLECACHE_PATH = "v1/cloudflare-skills/skills.tar.gz";
@@ -66,18 +68,10 @@ fs.mkdirSync(SKILLS_DIR, { recursive: true });
 // Extract the tarball from .tmp/ into ./skills/.
 // The archive contains skills/<skill-name>/... so we strip the leading "skills/"
 // component and extract into SKILLS_DIR.
-const tar = spawn(
-	"tar",
-	["--strip-components=1", "-xz", "-C", SKILLS_DIR, "-f", tarballPath],
-	{ stdio: "inherit" },
-);
-
-const exitCode = await new Promise<number | null>((resolve) =>
-	tar.on("close", resolve),
-);
-
-if (exitCode !== 0) {
-	fail(`tar exited with code ${exitCode}`);
+try {
+	await extractTarGz(tarballPath, SKILLS_DIR, { stripComponents: 1 });
+} catch (err) {
+	fail(`tar extraction failed: ${(err as Error).message}`);
 }
 
 const cloudflareSkills = fs

package.json

@@ -5,14 +5,14 @@
 	"scripts": {
 		"preinstall": "npx --yes only-allow pnpm",
 		"astro": "astro",
-		"prebuild": "tsx bin/fetch-skills.ts",
+		"prebuild": "pnpm run fetch:assets",
 		"build": "astro build",
 		"build:incremental": "INCREMENTAL_BUILD=true astro build",
 		"typegen:worker": "wrangler types ./worker/worker-configuration.d.ts",
 		"check": "pnpm run check:astro && pnpm run check:worker",
 		"check:astro": "astro check --minimumFailingSeverity=hint",
 		"check:worker": "tsc --noEmit -p ./worker/tsconfig.json",
-		"predev": "tsx bin/fetch-skills.ts --soft",
+		"predev": "tsx bin/fetch-skills.ts --soft && tsx bin/fetch-openapi.ts --soft",
 		"dev": "astro dev",
 		"format": "pnpm run format:core:fix && pnpm run format:data:fix && pnpm run format:content:fix",
 		"format:check": "pnpm run format:core:check && pnpm run format:data:check && pnpm run format:content:check",
@@ -38,7 +38,9 @@
 		"flue:reset:local": "rm -rf .flue/.wrangler/state .flue/.wrangler/tmp .flue/dist/cloudflare_docs_flue/.wrangler/state && echo 'Cleared local flue dev state (Durable Objects + R2). Stop the dev server before running this.'",
 		"flue:evals": "tsx .flue/bin/run-evals.ts",
 		"lint": "eslint",
-		"prepare": "husky"
+		"prepare": "husky",
+		"prebuild:incremental": "pnpm run fetch:assets",
+		"fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts"
 	},
 	"devDependencies": {
 		"@actions/core": "3.0.1",

src/util/api.ts

@@ -1,36 +1,43 @@
 /**
  * OpenAPI schema loader for the APIRequest component.
  *
- * Fetches the Cloudflare API OpenAPI document from middlecache and dereferences
- * all `$ref`s. The file is cached to `.tmp/middlecache/` (gitignored) via
- * `downloadToDotTempIfNotPresent`, so the fetch only happens once per clean
- * checkout. Dereferenced result is memoized at module scope so the deref runs
- * once per build, not per component instance.
+ * The schema is fetched by `bin/fetch-openapi.ts` from the `prebuild` and
+ * `prebuild:incremental` hooks (see package.json). `getSchema` reads the local
+ * copy and fails loudly if it is missing, so a build invoked without the
+ * pre-step is caught early instead of silently downloading mid-render. The
+ * dereferenced result is memoized so the deref runs once per build, not per
+ * component instance.
  */
 import SwaggerParser from "@apidevtools/swagger-parser";
 import type { OpenAPI } from "openapi-types";
-import { downloadToDotTempIfNotPresent } from "./custom-loaders";
 import { readFile } from "node:fs/promises";
-import { fileURLToPath } from "node:url";
-import { join } from "node:path";
+import { getOpenApiJsonPath } from "./openapi-schema";
 
-const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/";
-const API_SCHEMAS_PATH = "v1/cloudflare-api-schemas/openapi.json";
+let schemaPromise: Promise<OpenAPI.Document> | undefined;
 
-let schema: OpenAPI.Document | undefined;
+const loadSchema = async (): Promise<OpenAPI.Document> => {
+	const openapiFile = getOpenApiJsonPath();
 
-export const getSchema = async () => {
-	if (!schema) {
-		await downloadToDotTempIfNotPresent(
-			`${MIDDLECACHE_BASE_URL}${API_SCHEMAS_PATH}`,
-			`middlecache/${API_SCHEMAS_PATH}`,
+	let raw: string;
+	try {
+		raw = await readFile(openapiFile, "utf8");
+	} catch (cause) {
+		throw new Error(
+			`OpenAPI schema not found at ${openapiFile}. Run \`pnpm run build\` (or \`pnpm run build:incremental\`) so the prebuild hook fetches it first.`,
+			{ cause },
 		);
-		const dotTmpPath = fileURLToPath(new URL("../../.tmp", import.meta.url));
-		const filePath = join(dotTmpPath, "middlecache", API_SCHEMAS_PATH);
-		const raw = await readFile(filePath, "utf8");
-
-		schema = await SwaggerParser.dereference(JSON.parse(raw));
 	}
 
-	return schema;
+	return await SwaggerParser.dereference(JSON.parse(raw));
+};
+
+/**
+ * Load (and cache) the Cloudflare API OpenAPI document. Prerender renders
+ * pages in parallel, so this is single-flighted to avoid duplicate derefs.
+ */
+export const getSchema = (): Promise<OpenAPI.Document> => {
+	if (!schemaPromise) {
+		schemaPromise = loadSchema();
+	}
+	return schemaPromise;
 };