[pull] main from firecrawl:main

apps/api/src/__tests__/pdf-parser-options.test.ts

@@ -1,6 +1,7 @@
 import {
   getPDFBlocks,
   getPDFPageMarkdown,
+  getPDFPageMarkers,
   scrapeOptions,
 } from "../controllers/v2/types";
 
@@ -23,6 +24,27 @@ describe("PDF parser option getters", () => {
     expect(getPDFBlocks([{ type: "pdf" }])).toBe(false);
     expect(getPDFBlocks(undefined)).toBe(false);
   });
+
+  it("reads the `pageMarkers` option", () => {
+    expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: true }])).toBe(true);
+    expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: false }])).toBe(
+      false,
+    );
+    expect(getPDFPageMarkers([{ type: "pdf" }])).toBe(false);
+    expect(getPDFPageMarkers(["pdf"])).toBe(false);
+    expect(getPDFPageMarkers(undefined)).toBe(false);
+  });
+
+  it("keeps `pageMarkers` flowing through the alias-normalizing transform", () => {
+    const parser = parsePdfParser({
+      type: "pdf",
+      pageMarkers: true,
+      pageMarkdown: true,
+    });
+    expect(parser.pageMarkers).toBe(true);
+    expect(parser.pages).toBe(true);
+    expect(getPDFPageMarkers([parser as any])).toBe(true);
+  });
 });
 
 describe("deprecated pageMarkdown alias", () => {

apps/api/src/controllers/v2/types.ts

@@ -497,6 +497,13 @@ const pdfParserWithOptions = z
      * reading order) alongside document markdown — populates
      * `document.blocks`. */
     blocks: z.boolean().optional(),
+    /** Join PDF pages in `document.markdown` with
+     * `\n\n---\n\n<!-- page N -->\n\n` where N is the 1-based physical page
+     * of the content that follows. Markers appear between pages only (no
+     * leading marker for page 1), and numbering may skip pages merged by
+     * cross-page stitching — callers that need every physical page should
+     * use `pages: true` instead. No new response field. */
+    pageMarkers: z.boolean().optional(),
     // Experimental: route this request through the fire-pdf async pipeline
     // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
     // to sync on any async-path failure, so user-visible behavior is unchanged
@@ -583,6 +590,16 @@ export function getPDFBlocks(parsers?: Parsers): boolean {
   return false;
 }
 
+export function getPDFPageMarkers(parsers?: Parsers): boolean {
+  if (!parsers) return false;
+  for (const parser of parsers) {
+    if (typeof parser === "object" && parser.type === "pdf") {
+      return parser.pageMarkers === true;
+    }
+  }
+  return false;
+}
+
 export function getFirePdfAsync(parsers?: Parsers): boolean {
   if (!parsers) return false;
   for (const parser of parsers) {

apps/api/src/scraper/scrapeURL/engines/index.ts

@@ -30,6 +30,7 @@ import { hasFormatOfType } from "../../../lib/format-utils";
 import {
   getPDFBlocks,
   getPDFPageMarkdown,
+  getPDFPageMarkers,
 } from "../../../controllers/v2/types";
 import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";
 import { BrandingProfile } from "../../../types/branding";
@@ -567,9 +568,10 @@ export function shouldUseIndex(meta: Meta) {
     !hasFormatOfType(meta.options.formats, "changeTracking") &&
     !hasFormatOfType(meta.options.formats, "branding") &&
     // The URL index does not yet persist physical-page or typed-block
-    // capability metadata.
+    // capability metadata, and its markdown never carries page markers.
     !getPDFPageMarkdown(meta.options.parsers) &&
     !getPDFBlocks(meta.options.parsers) &&
+    !getPDFPageMarkers(meta.options.parsers) &&
     !hasCustomScreenshotSettings &&
     meta.options.maxAge !== 0 &&
     (meta.options.headers === undefined ||
@@ -680,17 +682,19 @@ export async function buildFallbackList(meta: Meta): Promise<
     _engines.push(...indexEngines);
     meta.internalOptions.forceEngine = indexEngines;
   } else if (meta.internalOptions.agentIndexOnly) {
-    // Index documents carry no physical-page or typed-block payloads, so an
-    // index-only request that demands them can only be answered wrong. Fail
-    // loud with the canonical index-only error (maps to a clean 4xx and
-    // tells the caller how to unlock live scraping) instead of silently
-    // serving a document without the capability.
+    // Index documents carry no physical-page or typed-block payloads, and
+    // their markdown never carries page markers, so an index-only request
+    // that demands them can only be answered wrong. Fail loud with the
+    // canonical index-only error (maps to a clean 4xx and tells the caller
+    // how to unlock live scraping) instead of silently serving a document
+    // without the capability.
     if (
       getPDFPageMarkdown(meta.options.parsers) ||
-      getPDFBlocks(meta.options.parsers)
+      getPDFBlocks(meta.options.parsers) ||
+      getPDFPageMarkers(meta.options.parsers)
     ) {
       meta.logger.warn(
-        "agentIndexOnly request demands pageMarkdown/blocks, which the URL index cannot serve",
+        "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve",
         { parsers: meta.options.parsers },
       );
       throw new AgentIndexOnlyError();

apps/api/src/scraper/scrapeURL/engines/index/index.ts

@@ -40,6 +40,7 @@ import {
   getPDFBlocks,
   getPDFMaxPages,
   getPDFPageMarkdown,
+  getPDFPageMarkers,
   shouldParsePDF,
 } from "../../../../controllers/v2/types";
 import { hasFormatOfType } from "../../../../lib/format-utils";
@@ -64,8 +65,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
     // Page-aware and block-aware results are capability-specific and are not
     // represented in the URL index schema yet. Do not write an entry that
     // could later be served without its required pages/blocks payload.
+    // Marker-bearing markdown is mutated output — never index it either.
     !getPDFPageMarkdown(meta.options.parsers) &&
     !getPDFBlocks(meta.options.parsers) &&
+    !getPDFPageMarkers(meta.options.parsers) &&
     !meta.options.parsers?.some(parser => {
       if (
         typeof parser === "object" &&

apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts

@@ -211,3 +211,105 @@ describe("scrapePDFWithFirePDF typed blocks", () => {
     ).rejects.toThrow(/did not include requested typed blocks/);
   });
 });
+
+describe("scrapePDFWithFirePDF page markers", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("requests marker-joined markdown and returns it verbatim", async () => {
+    const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";
+    mockedRobustFetch.mockResolvedValue({
+      markdown: marked,
+      failed_pages: null,
+      pages_processed: 2,
+      page_markers: true,
+    } as any);
+
+    const result = await scrapePDFWithFirePDF(
+      makeMeta(),
+      "BASE64",
+      undefined,
+      undefined,
+      "auto",
+      false,
+      false,
+      true,
+    );
+
+    expect(mockedRobustFetch).toHaveBeenCalledWith(
+      expect.objectContaining({
+        body: expect.objectContaining({ page_markers: true }),
+      }),
+    );
+    expect(result.markdown).toBe(marked);
+  });
+
+  it("does not send page_markers for plain requests", async () => {
+    mockedRobustFetch.mockResolvedValue({
+      markdown: "plain",
+      failed_pages: null,
+      pages_processed: 1,
+    } as any);
+
+    await scrapePDFWithFirePDF(makeMeta(), "BASE64");
+
+    expect(
+      ((mockedRobustFetch.mock.calls[0][0] as any).body as any).page_markers,
+    ).toBeUndefined();
+  });
+
+  it("rejects FirePDF responses that do not acknowledge page markers", async () => {
+    // An old fire-pdf build ignores the unknown `page_markers` request field
+    // and returns ordinary markdown with no echo — indistinguishable from
+    // marked output by content alone, so the missing echo must fail loud.
+    mockedRobustFetch.mockResolvedValue({
+      markdown: "Page 1\n\n---\n\nPage 2",
+      failed_pages: null,
+      pages_processed: 2,
+    } as any);
+
+    await expect(
+      scrapePDFWithFirePDF(
+        makeMeta(),
+        "BASE64",
+        undefined,
+        undefined,
+        "auto",
+        false,
+        false,
+        true,
+      ),
+    ).rejects.toThrow(/did not acknowledge requested page markers/);
+  });
+
+  it("composes page_markers with include_blocks on the wire", async () => {
+    mockedRobustFetch.mockResolvedValue({
+      markdown: "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2",
+      failed_pages: null,
+      pages_processed: 2,
+      blocks: [],
+      page_markers: true,
+    } as any);
+
+    await scrapePDFWithFirePDF(
+      makeMeta(),
+      "BASE64",
+      undefined,
+      undefined,
+      "auto",
+      false,
+      true,
+      true,
+    );
+
+    expect(mockedRobustFetch).toHaveBeenCalledWith(
+      expect.objectContaining({
+        body: expect.objectContaining({
+          include_blocks: true,
+          page_markers: true,
+        }),
+      }),
+    );
+  });
+});

apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts

@@ -348,6 +348,99 @@ describe("scrapePDFWithFirePDFAsync", () => {
     expect(error.reason).toBe("http_5xx");
   });
 
+  it("requests page markers and returns the acknowledged marked markdown", async () => {
+    const marked = "Page 1\n\n---\n\n<!-- page 2 -->\n\nPage 2";
+    const { fetchImpl, calls } = makeFetchFromSequence([
+      {
+        matchUrl: /\/jobs$/,
+        matchMethod: "POST",
+        response: {
+          status: 200,
+          body: {
+            scrape_id: "scrape-id-test",
+            status: "done",
+            lane: "fast",
+          },
+        },
+      },
+      {
+        matchUrl: /\/jobs\/scrape-id-test\/result$/,
+        matchMethod: "GET",
+        response: {
+          status: 200,
+          body: {
+            schema_version: 1,
+            markdown: marked,
+            pages_processed: 2,
+            page_markers: true,
+          },
+        },
+      },
+    ]);
+
+    const result = await scrapePDFWithFirePDFAsync(
+      makeMeta(),
+      "BASE64",
+      undefined,
+      undefined,
+      "auto",
+      { fetchImpl, sleepImpl: noopSleep },
+      false,
+      false,
+      true,
+    );
+
+    expect((calls[0].body as any).options.pageMarkers).toBe(true);
+    expect(result.markdown).toBe(marked);
+  });
+
+  it("fails a marker request when the result lacks the page_markers echo", async () => {
+    // An older worker ignores the unknown pageMarkers option and persists
+    // ordinary markdown; without the echo this must fail (and fall back to
+    // the sync path) rather than cache unmarked markdown as marked output.
+    const { fetchImpl } = makeFetchFromSequence([
+      {
+        matchUrl: /\/jobs$/,
+        matchMethod: "POST",
+        response: {
+          status: 200,
+          body: {
+            scrape_id: "scrape-id-test",
+            status: "done",
+            lane: "fast",
+          },
+        },
+      },
+      {
+        matchUrl: /\/jobs\/scrape-id-test\/result$/,
+        matchMethod: "GET",
+        response: {
+          status: 200,
+          body: {
+            schema_version: 1,
+            markdown: "Page 1\n\n---\n\nPage 2",
+            pages_processed: 2,
+          },
+        },
+      },
+    ]);
+
+    const error = await scrapePDFWithFirePDFAsync(
+      makeMeta(),
+      "BASE64",
+      undefined,
+      undefined,
+      "auto",
+      { fetchImpl, sleepImpl: noopSleep },
+      false,
+      false,
+      true,
+    ).catch(error => error);
+
+    expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+    expect(error.reason).toBe("http_5xx");
+  });
+
   it("requests and returns typed blocks, tolerating the legacy pages alias", async () => {
     const blocks = [
       {