diff --git a/bin/fetch-openapi.ts b/bin/fetch-openapi.ts new file mode 100644 index 00000000000..485d7072a4d --- /dev/null +++ b/bin/fetch-openapi.ts @@ -0,0 +1,44 @@ +#!/usr/bin/env tsx + +import fs from "fs"; + +import { + fetchOpenApiSchema, + getOpenApiJsonPath, +} from "../src/util/openapi-schema"; + +// --soft: warn and continue on failure instead of exiting non-zero. +// Used by the predev hook so a network failure doesn't block local development. +// --force: re-fetch even if the schema already exists. +const soft = process.argv.includes("--soft"); +const force = process.argv.includes("--force"); + +const fail = (message: string): never => { + if (soft) { + console.warn( + `Warning: ${message} — API endpoint pages will not work without the schema`, + ); + process.exit(0); + } + console.error(`Error: ${message}`); + process.exit(1); +}; + +const openapiFile = getOpenApiJsonPath(); + +if (fs.existsSync(openapiFile) && !force) { + console.log( + "OpenAPI schema already exists, skipping fetch. (run `pnpm tsx bin/fetch-openapi.ts --force` to re-fetch)", + ); + process.exit(0); +} + +console.log("Fetching Cloudflare API OpenAPI schema from middlecache"); + +try { + await fetchOpenApiSchema(); +} catch (err) { + fail(`fetch failed: ${err}`); +} + +console.log("OpenAPI schema ready"); diff --git a/bin/fetch-skills.ts b/bin/fetch-skills.ts index db2109bc7b8..ba0d648c6d3 100644 --- a/bin/fetch-skills.ts +++ b/bin/fetch-skills.ts @@ -1,10 +1,12 @@ #!/usr/bin/env tsx -import { spawn } from "child_process"; import fs from "fs"; import { join } from "path"; -import { downloadToDotTempIfNotPresent } from "../src/util/custom-loaders"; +import { + downloadToDotTempIfNotPresent, + extractTarGz, +} from "../src/util/custom-loaders"; const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/"; const SKILLS_MIDDLECACHE_PATH = "v1/cloudflare-skills/skills.tar.gz"; @@ -66,18 +68,10 @@ fs.mkdirSync(SKILLS_DIR, { recursive: true }); // Extract the tarball from .tmp/ into ./skills/. // The archive contains skills//... so we strip the leading "skills/" // component and extract into SKILLS_DIR. -const tar = spawn( - "tar", - ["--strip-components=1", "-xz", "-C", SKILLS_DIR, "-f", tarballPath], - { stdio: "inherit" }, -); - -const exitCode = await new Promise((resolve) => - tar.on("close", resolve), -); - -if (exitCode !== 0) { - fail(`tar exited with code ${exitCode}`); +try { + await extractTarGz(tarballPath, SKILLS_DIR, { stripComponents: 1 }); +} catch (err) { + fail(`tar extraction failed: ${(err as Error).message}`); } const cloudflareSkills = fs diff --git a/package.json b/package.json index aff278e00d9..14f79723422 100644 --- a/package.json +++ b/package.json @@ -5,14 +5,14 @@ "scripts": { "preinstall": "npx --yes only-allow pnpm", "astro": "astro", - "prebuild": "tsx bin/fetch-skills.ts", + "prebuild": "pnpm run fetch:assets", "build": "astro build", "build:incremental": "INCREMENTAL_BUILD=true astro build", "typegen:worker": "wrangler types ./worker/worker-configuration.d.ts", "check": "pnpm run check:astro && pnpm run check:worker", "check:astro": "astro check --minimumFailingSeverity=hint", "check:worker": "tsc --noEmit -p ./worker/tsconfig.json", - "predev": "tsx bin/fetch-skills.ts --soft", + "predev": "tsx bin/fetch-skills.ts --soft && tsx bin/fetch-openapi.ts --soft", "dev": "astro dev", "format": "pnpm run format:core:fix && pnpm run format:data:fix && pnpm run format:content:fix", "format:check": "pnpm run format:core:check && pnpm run format:data:check && pnpm run format:content:check", @@ -38,7 +38,9 @@ "flue:reset:local": "rm -rf .flue/.wrangler/state .flue/.wrangler/tmp .flue/dist/cloudflare_docs_flue/.wrangler/state && echo 'Cleared local flue dev state (Durable Objects + R2). Stop the dev server before running this.'", "flue:evals": "tsx .flue/bin/run-evals.ts", "lint": "eslint", - "prepare": "husky" + "prepare": "husky", + "prebuild:incremental": "pnpm run fetch:assets", + "fetch:assets": "tsx bin/fetch-skills.ts && tsx bin/fetch-openapi.ts" }, "devDependencies": { "@actions/core": "3.0.1", diff --git a/src/util/api.ts b/src/util/api.ts index 98b458c133a..3752547a5b7 100644 --- a/src/util/api.ts +++ b/src/util/api.ts @@ -1,36 +1,43 @@ /** * OpenAPI schema loader for the APIRequest component. * - * Fetches the Cloudflare API OpenAPI document from middlecache and dereferences - * all `$ref`s. The file is cached to `.tmp/middlecache/` (gitignored) via - * `downloadToDotTempIfNotPresent`, so the fetch only happens once per clean - * checkout. Dereferenced result is memoized at module scope so the deref runs - * once per build, not per component instance. + * The schema is fetched by `bin/fetch-openapi.ts` from the `prebuild` and + * `prebuild:incremental` hooks (see package.json). `getSchema` reads the local + * copy and fails loudly if it is missing, so a build invoked without the + * pre-step is caught early instead of silently downloading mid-render. The + * dereferenced result is memoized so the deref runs once per build, not per + * component instance. */ import SwaggerParser from "@apidevtools/swagger-parser"; import type { OpenAPI } from "openapi-types"; -import { downloadToDotTempIfNotPresent } from "./custom-loaders"; import { readFile } from "node:fs/promises"; -import { fileURLToPath } from "node:url"; -import { join } from "node:path"; +import { getOpenApiJsonPath } from "./openapi-schema"; -const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/"; -const API_SCHEMAS_PATH = "v1/cloudflare-api-schemas/openapi.json"; +let schemaPromise: Promise | undefined; -let schema: OpenAPI.Document | undefined; +const loadSchema = async (): Promise => { + const openapiFile = getOpenApiJsonPath(); -export const getSchema = async () => { - if (!schema) { - await downloadToDotTempIfNotPresent( - `${MIDDLECACHE_BASE_URL}${API_SCHEMAS_PATH}`, - `middlecache/${API_SCHEMAS_PATH}`, + let raw: string; + try { + raw = await readFile(openapiFile, "utf8"); + } catch (cause) { + throw new Error( + `OpenAPI schema not found at ${openapiFile}. Run \`pnpm run build\` (or \`pnpm run build:incremental\`) so the prebuild hook fetches it first.`, + { cause }, ); - const dotTmpPath = fileURLToPath(new URL("../../.tmp", import.meta.url)); - const filePath = join(dotTmpPath, "middlecache", API_SCHEMAS_PATH); - const raw = await readFile(filePath, "utf8"); - - schema = await SwaggerParser.dereference(JSON.parse(raw)); } - return schema; + return await SwaggerParser.dereference(JSON.parse(raw)); +}; + +/** + * Load (and cache) the Cloudflare API OpenAPI document. Prerender renders + * pages in parallel, so this is single-flighted to avoid duplicate derefs. + */ +export const getSchema = (): Promise => { + if (!schemaPromise) { + schemaPromise = loadSchema(); + } + return schemaPromise; }; diff --git a/src/util/custom-loaders.node.test.ts b/src/util/custom-loaders.node.test.ts new file mode 100644 index 00000000000..25d8650708c --- /dev/null +++ b/src/util/custom-loaders.node.test.ts @@ -0,0 +1,327 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import fs from "fs"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; +import zlib from "node:zlib"; + +import { + downloadToDotTempIfNotPresent, + extractTarGz, + getDotTmpPath, +} from "./custom-loaders"; + +const dotTmpPath = getDotTmpPath(); +const TEST_DIR = join(dotTmpPath, "middlecache", "__custom-loaders-test__"); +const TEST_DEST = "middlecache/__custom-loaders-test__/file.txt"; + +const okResponse = (body: string) => new Response(body); +const errResponse = (status: number) => new Response("error", { status }); + +// Minimal ustar header so a test can craft an archive with an arbitrary +// (potentially unsafe) member path, which the system `tar` refuses to create. +const ustarHeader = (name: string, size: number): Buffer => { + const header = Buffer.alloc(512); + header.write(name, 0, 100, "utf8"); + header.write("0000644\0", 100, 8, "ascii"); + header.write("0000000\0", 108, 8, "ascii"); + header.write("0000000\0", 116, 8, "ascii"); + header.write(size.toString(8).padStart(11, "0") + "\0", 124, 12, "ascii"); + header.write("00000000000\0", 136, 12, "ascii"); + header.fill(0x20, 148, 156); + header.write("0", 156, 1, "ascii"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, "0"), 148, 6, "ascii"); + header[154] = 0; + header[155] = 0x20; + return header; +}; + +const buildUnsafeArchive = (tarballPath: string, memberPath: string) => { + const body = Buffer.from("evil"); + const block = Buffer.alloc(512); + body.copy(block); + const raw = Buffer.concat([ + ustarHeader(memberPath, body.length), + block, + Buffer.alloc(1024), + ]); + fs.writeFileSync(tarballPath, zlib.gzipSync(raw)); +}; + +describe("downloadToDotTempIfNotPresent", () => { + beforeEach(() => { + fs.rmSync(TEST_DIR, { recursive: true, force: true }); + }); + + afterEach(() => { + fs.rmSync(TEST_DIR, { recursive: true, force: true }); + vi.unstubAllGlobals(); + }); + + test("downloads and writes the file when fetch succeeds", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(okResponse("hello world")), + ); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + ); + + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe( + "hello world", + ); + }); + + test("retries on HTTP error then succeeds", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(errResponse(500)) + .mockResolvedValueOnce(errResponse(500)) + .mockResolvedValueOnce(okResponse("ok")); + vi.stubGlobal("fetch", fetchMock); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + ); + + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe("ok"); + }); + + test("throws after exhausting retries", async () => { + const fetchMock = vi.fn().mockResolvedValue(errResponse(500)); + vi.stubGlobal("fetch", fetchMock); + + await expect( + downloadToDotTempIfNotPresent("https://example.com/file.txt", TEST_DEST), + ).rejects.toThrow(/HTTP 500/); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + test("rejects when the response has no body", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 200 })), + ); + + await expect( + downloadToDotTempIfNotPresent("https://example.com/file.txt", TEST_DEST), + ).rejects.toThrow(/Missing response body/); + }); + + test("re-downloads when the validate callback rejects", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(okResponse("corrupt")) + .mockResolvedValueOnce(okResponse("valid")); + vi.stubGlobal("fetch", fetchMock); + const validate = vi + .fn() + .mockRejectedValueOnce(new Error("corrupt file")) + .mockResolvedValueOnce(undefined); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + { + validate, + }, + ); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe("valid"); + }); + + test("retries when downloaded size mismatches content-length", async () => { + const lengthMismatch = () => { + // Real body ("abc", 3 bytes) but a fake content-length header (10). + const res = new Response("abc"); + return { + ok: true, + status: 200, + statusText: "OK", + body: res.body, + headers: { get: () => "10" }, + }; + }; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(lengthMismatch()) + .mockResolvedValueOnce(okResponse("ok")); + vi.stubGlobal("fetch", fetchMock); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + ); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe("ok"); + }); + + test("skips download when the file already exists and validates", async () => { + fs.mkdirSync(TEST_DIR, { recursive: true }); + fs.writeFileSync(join(TEST_DIR, "file.txt"), "existing"); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + ); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + test("re-downloads when an existing file fails validation", async () => { + fs.mkdirSync(TEST_DIR, { recursive: true }); + fs.writeFileSync(join(TEST_DIR, "file.txt"), "stale"); + const fetchMock = vi.fn().mockResolvedValue(okResponse("fresh")); + vi.stubGlobal("fetch", fetchMock); + const validate = vi + .fn() + .mockRejectedValueOnce(new Error("bad file")) + .mockResolvedValueOnce(undefined); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + { + validate, + }, + ); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe("fresh"); + }); + + test("concurrent calls to the same destination share a single download", async () => { + const fetchMock = vi.fn().mockResolvedValue(okResponse("shared")); + vi.stubGlobal("fetch", fetchMock); + + await Promise.all([ + downloadToDotTempIfNotPresent("https://example.com/file.txt", TEST_DEST), + downloadToDotTempIfNotPresent("https://example.com/file.txt", TEST_DEST), + downloadToDotTempIfNotPresent("https://example.com/file.txt", TEST_DEST), + ]); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe("shared"); + }); + + test("a failed in-flight download can be retried by a later call", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(errResponse(500)) + .mockResolvedValueOnce(errResponse(500)) + .mockResolvedValueOnce(errResponse(500)) + .mockResolvedValueOnce(okResponse("ok")); + vi.stubGlobal("fetch", fetchMock); + + await expect( + downloadToDotTempIfNotPresent("https://example.com/file.txt", TEST_DEST), + ).rejects.toThrow(/HTTP 500/); + + await downloadToDotTempIfNotPresent( + "https://example.com/file.txt", + TEST_DEST, + ); + + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fs.readFileSync(join(TEST_DIR, "file.txt"), "utf8")).toBe("ok"); + }); +}); + +describe("extractTarGz", () => { + const FIXTURE = join(dotTmpPath, "middlecache", "__extract-test__"); + + const buildFixture = () => { + fs.mkdirSync(join(FIXTURE, "src", "skills", "skill-name"), { + recursive: true, + }); + fs.writeFileSync( + join(FIXTURE, "src", "skills", "skill-name", "file.txt"), + "content", + ); + const result = spawnSync( + "tar", + [ + "-czf", + join(FIXTURE, "fixture.tar.gz"), + "-C", + join(FIXTURE, "src"), + "skills", + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + if (result.status !== 0 || result.error) { + throw new Error( + `Failed to build test fixture archive: ${ + result.error?.message ?? "" + } ${result.stderr?.toString() ?? ""}`.trim(), + ); + } + }; + + beforeEach(() => { + fs.rmSync(FIXTURE, { recursive: true, force: true }); + }); + + afterEach(() => { + fs.rmSync(FIXTURE, { recursive: true, force: true }); + }); + + test("extracts without stripping components", async () => { + buildFixture(); + const dest = join(FIXTURE, "out"); + + await extractTarGz(join(FIXTURE, "fixture.tar.gz"), dest); + + expect( + fs.readFileSync(join(dest, "skills", "skill-name", "file.txt"), "utf8"), + ).toBe("content"); + }); + + test("strips leading components when stripComponents is set", async () => { + buildFixture(); + const dest = join(FIXTURE, "out"); + + await extractTarGz(join(FIXTURE, "fixture.tar.gz"), dest, { + stripComponents: 1, + }); + + expect(fs.readFileSync(join(dest, "skill-name", "file.txt"), "utf8")).toBe( + "content", + ); + }); + + test("throws on a corrupt archive", async () => { + fs.mkdirSync(FIXTURE, { recursive: true }); + fs.writeFileSync(join(FIXTURE, "bad.tar.gz"), "not a gzip archive"); + + await expect( + extractTarGz(join(FIXTURE, "bad.tar.gz"), join(FIXTURE, "out")), + ).rejects.toThrow(/tar extraction failed/); + }); + + test("rejects archive members that escape the destination directory", async () => { + fs.mkdirSync(FIXTURE, { recursive: true }); + buildUnsafeArchive(join(FIXTURE, "escape.tar.gz"), "../evil.txt"); + + await expect( + extractTarGz(join(FIXTURE, "escape.tar.gz"), join(FIXTURE, "out")), + ).rejects.toThrow(/unsafe member path/); + }); + + test("rejects archive members with absolute paths", async () => { + fs.mkdirSync(FIXTURE, { recursive: true }); + buildUnsafeArchive(join(FIXTURE, "escape.tar.gz"), "/etc/evil.txt"); + + await expect( + extractTarGz(join(FIXTURE, "escape.tar.gz"), join(FIXTURE, "out")), + ).rejects.toThrow(/unsafe member path/); + }); +}); diff --git a/src/util/custom-loaders.ts b/src/util/custom-loaders.ts index 7cc3c068e0d..8562c89eb3d 100644 --- a/src/util/custom-loaders.ts +++ b/src/util/custom-loaders.ts @@ -7,22 +7,70 @@ import { fileURLToPath } from "node:url"; import { Readable } from "node:stream"; import type { ReadableStream as WebReadableStream } from "node:stream/web"; import { writeFile } from "node:fs/promises"; +import { spawn, spawnSync } from "node:child_process"; import fs from "fs"; import { dirname, join } from "path"; import * as z from "zod"; +const MAX_DOWNLOAD_ATTEMPTS = 3; + +// Serialize concurrent downloads of the same destination. Prerender renders +// pages in parallel, so multiple callers can hit the same middlecache file at +// once; without this they race on the temp-file write/rename and a retry's +// cleanup can delete a sibling call's freshly-written output. +const inFlightDownloads = new Map>(); + +/** + * Resolve the repo-root `.tmp/` directory used for downloaded artifacts. + * + * The tsx prebuild scripts and the bundled prerender resolve `import.meta.url` + * to different locations (source files vs `dist/.prerender/chunks/`), so the + * repo root is found by walking up from the module location until a + * `package.json` is found. Falls back to the current working directory for + * runtimes where `import.meta.url` is not a `file://` URL (e.g. Vitest). + */ +export const getDotTmpPath = () => { + try { + const moduleDir = dirname(fileURLToPath(import.meta.url)); + const root = findRepoRoot(moduleDir); + if (root) { + return join(root, ".tmp"); + } + } catch { + // not a file:// URL (e.g. under Vitest) + } + return join(process.cwd(), ".tmp"); +}; + +const findRepoRoot = (startDir: string): string | undefined => { + let dir = startDir; + for (;;) { + if (fs.existsSync(join(dir, "package.json"))) { + return dir; + } + const parent = dirname(dir); + if (parent === dir) { + return undefined; + } + dir = parent; + } +}; + /** * downloadToDotTempIfNotPresent is a convenience function for handling downloads to a .tmp directory * within the source repo * * @param url - source URL * @param dotTmpDestination - path relative to .tmp/ as destination for downloaded file + * @param options - { validate: optional async check run against the downloaded file; a rejected + * promise discards the file and triggers a re-download } */ export async function downloadToDotTempIfNotPresent( url: string, dotTmpDestination: string, + options: { validate?: (filePath: string) => Promise } = {}, ) { const source = z.url().parse(url); const relativeDestination = z @@ -38,26 +86,153 @@ export async function downloadToDotTempIfNotPresent( const destinationParts = relativeDestination.split("/"); const universalRelativeDestination = join(...destinationParts); - const dotTmpPath = fileURLToPath(new URL("../../.tmp", import.meta.url)); + const destination = join(getDotTmpPath(), universalRelativeDestination); - const destination = join(dotTmpPath, universalRelativeDestination); + const inFlight = inFlightDownloads.get(destination); + if (inFlight) { + return inFlight; + } - if (!fs.existsSync(destination)) { - fs.mkdirSync(dirname(destination), { recursive: true }); + const promise = downloadWithRetry(source, url, destination, options.validate); + inFlightDownloads.set(destination, promise); + try { + await promise; + } finally { + inFlightDownloads.delete(destination); + } +} - const response = await fetch(source); +const downloadWithRetry = async ( + source: string, + url: string, + destination: string, + validate: ((filePath: string) => Promise) | undefined, +) => { + for (let attempt = 1; attempt <= MAX_DOWNLOAD_ATTEMPTS; attempt++) { try { + if (fs.existsSync(destination)) { + await validate?.(destination); + return; + } + + fs.mkdirSync(dirname(destination), { recursive: true }); + + // Write to a temp file first so a partial/failed download never + // leaves a file that looks "present". + const tmpDestination = `${destination}.tmp`; + fs.rmSync(tmpDestination, { force: true }); + + // Request the identity encoding so middlecache serves the bytes + // as-is rather than on-the-fly brotli, which has no integrity check + // and can silently decompress a truncated transfer into garbage. + const response = await fetch(source, { + headers: { "Accept-Encoding": "identity" }, + }); + + if (!response.ok) { + throw new Error( + `Failed to download ${url}: HTTP ${response.status} ${response.statusText}`, + ); + } + + if (!response.body) { + throw new Error(`Missing response body for ${url}`); + } + // Stream file to destination to avoid storing in memory await writeFile( - destination, - Readable.fromWeb(response.body! as WebReadableStream), + tmpDestination, + Readable.fromWeb(response.body as WebReadableStream), ); + + const expectedLength = Number(response.headers.get("content-length")); + if (Number.isFinite(expectedLength) && expectedLength > 0) { + const actualLength = fs.statSync(tmpDestination).size; + if (actualLength !== expectedLength) { + throw new Error( + `Downloaded file size mismatch for ${url}: expected ${expectedLength} bytes, got ${actualLength}`, + ); + } + } + + fs.renameSync(tmpDestination, destination); + await validate?.(destination); + return; } catch (err) { - // Clean up partial download if stream fails fs.rmSync(destination, { force: true }); - throw err; + fs.rmSync(`${destination}.tmp`, { force: true }); + if (attempt === MAX_DOWNLOAD_ATTEMPTS) { + throw err; + } + console.warn( + `Retrying download of ${url} (attempt ${attempt}/${MAX_DOWNLOAD_ATTEMPTS}): ${(err as Error).message}`, + ); + } + } +}; + +/** + * Extract a gzip-compressed tar archive into destinationDir. + * + * Member paths are validated before extraction: entries containing `..` + * segments or leading `/` are rejected (zip-slip) so a network-downloaded + * archive cannot write outside destinationDir. Extraction failures reject + * with captured stderr instead of hanging. + * + * @param options.stripComponents - strip the given number of leading path + * components from each entry before extracting (matches the skills archive, + * which contains a top-level `skills/` directory). + */ +export async function extractTarGz( + tarballPath: string, + destinationDir: string, + options: { stripComponents?: number } = {}, +): Promise { + // Refuse archives whose members could escape destinationDir. + const list = spawnSync("tar", ["-tzf", tarballPath], { encoding: "utf8" }); + if (list.status !== 0 || list.error) { + throw new Error( + `tar extraction failed for ${tarballPath}: not a valid archive${ + list.stderr ? `: ${list.stderr.trim()}` : "" + }`, + ); + } + for (const member of list.stdout.split("\n")) { + const name = member.trimEnd(); + if (!name) { + continue; + } + if (name.startsWith("/") || name.split("/").includes("..")) { + throw new Error( + `tar extraction failed for ${tarballPath}: refusing unsafe member path "${name}"`, + ); } } + + fs.mkdirSync(destinationDir, { recursive: true }); + + const args = ["-xz", "-C", destinationDir]; + if (options.stripComponents && options.stripComponents > 0) { + args.push(`--strip-components=${options.stripComponents}`); + } + args.push("-f", tarballPath); + + const tar = spawn("tar", args, { stdio: ["ignore", "ignore", "pipe"] }); + const stderr: Buffer[] = []; + tar.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk)); + const exitCode = await new Promise((resolve, reject) => { + tar.on("error", reject); + tar.on("close", resolve); + }); + if (exitCode !== 0) { + throw new Error( + `tar extraction failed for ${tarballPath} (exit code ${exitCode})${ + stderr.length + ? `: ${Buffer.concat(stderr).toString("utf8").trim()}` + : "" + }`, + ); + } } /** diff --git a/src/util/openapi-schema.ts b/src/util/openapi-schema.ts new file mode 100644 index 00000000000..0ffd5ab5fa9 --- /dev/null +++ b/src/util/openapi-schema.ts @@ -0,0 +1,86 @@ +/** + * Download and extract the Cloudflare API OpenAPI schema from middlecache. + * + * Used by `bin/fetch-openapi.ts`, which runs from the `prebuild`, + * `prebuild:incremental`, and `predev` hooks (see package.json). + */ +import { mkdir, readFile, rename, rm } from "node:fs/promises"; +import { join } from "node:path"; + +import { + downloadToDotTempIfNotPresent, + extractTarGz, + getDotTmpPath, +} from "./custom-loaders"; + +const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/"; +const OPENAPI_ARCHIVE_PATH = "v1/cloudflare-api-schemas/openapi.tar.gz"; +const OPENAPI_JSON_PATH = "v1/cloudflare-api-schemas/openapi.json"; + +export const getOpenApiExtractDir = () => + join(getDotTmpPath(), "middlecache", "v1", "cloudflare-api-schemas"); + +export const getOpenApiJsonPath = () => + join(getOpenApiExtractDir(), "openapi.json"); + +/** + * Download the schema into `.tmp` and return the path to `openapi.json`. + * + * Prefers the gzip-compressed archive (integrity-checkable via gzip CRC32), + * falling back to the raw openapi.json. Downloads are validated (extract + + * parse) and retried by `downloadToDotTempIfNotPresent`. + */ +export const fetchOpenApiSchema = async (): Promise => { + const extractDir = getOpenApiExtractDir(); + + try { + await downloadToDotTempIfNotPresent( + `${MIDDLECACHE_BASE_URL}${OPENAPI_ARCHIVE_PATH}`, + `middlecache/${OPENAPI_ARCHIVE_PATH}`, + { + validate: async (archivePath) => { + // Extract into a staging directory, then promote it on success + // so a failed extract or parse never leaves stale or partial + // files in the destination that could mask a fresh failure. + const stagingDir = join( + getDotTmpPath(), + "middlecache", + "v1", + ".openapi-staging", + ); + await rm(stagingDir, { recursive: true, force: true }); + try { + await mkdir(stagingDir, { recursive: true }); + await extractTarGz(archivePath, stagingDir); + const raw = await readFile( + join(stagingDir, "openapi.json"), + "utf8", + ); + void JSON.parse(raw); + await rm(extractDir, { recursive: true, force: true }); + await rename(stagingDir, extractDir); + } catch (err) { + await rm(stagingDir, { recursive: true, force: true }); + throw err; + } + }, + }, + ); + } catch (err) { + console.warn( + `Failed to fetch OpenAPI archive, falling back to raw openapi.json: ${(err as Error).message}`, + ); + await downloadToDotTempIfNotPresent( + `${MIDDLECACHE_BASE_URL}${OPENAPI_JSON_PATH}`, + `middlecache/${OPENAPI_JSON_PATH}`, + { + validate: async (filePath) => { + const raw = await readFile(filePath, "utf8"); + void JSON.parse(raw); + }, + }, + ); + } + + return getOpenApiJsonPath(); +};