diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.ts
index f891efe6e9..d3f783f715 100644
--- a/apps/api/src/__tests__/pdf-parser-options.test.ts
+++ b/apps/api/src/__tests__/pdf-parser-options.test.ts
@@ -1,6 +1,7 @@
import {
getPDFBlocks,
getPDFPageMarkdown,
+ getPDFPageMarkers,
scrapeOptions,
} from "../controllers/v2/types";
@@ -23,6 +24,27 @@ describe("PDF parser option getters", () => {
expect(getPDFBlocks([{ type: "pdf" }])).toBe(false);
expect(getPDFBlocks(undefined)).toBe(false);
});
+
+ it("reads the `pageMarkers` option", () => {
+ expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: true }])).toBe(true);
+ expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: false }])).toBe(
+ false,
+ );
+ expect(getPDFPageMarkers([{ type: "pdf" }])).toBe(false);
+ expect(getPDFPageMarkers(["pdf"])).toBe(false);
+ expect(getPDFPageMarkers(undefined)).toBe(false);
+ });
+
+ it("keeps `pageMarkers` flowing through the alias-normalizing transform", () => {
+ const parser = parsePdfParser({
+ type: "pdf",
+ pageMarkers: true,
+ pageMarkdown: true,
+ });
+ expect(parser.pageMarkers).toBe(true);
+ expect(parser.pages).toBe(true);
+ expect(getPDFPageMarkers([parser as any])).toBe(true);
+ });
});
describe("deprecated pageMarkdown alias", () => {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 54233c7991..496d00384f 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -497,6 +497,13 @@ const pdfParserWithOptions = z
* reading order) alongside document markdown — populates
* `document.blocks`. */
blocks: z.boolean().optional(),
+ /** Join PDF pages in `document.markdown` with
+ * `\n\n---\n\n\n\n` where N is the 1-based physical page
+ * of the content that follows. Markers appear between pages only (no
+ * leading marker for page 1), and numbering may skip pages merged by
+ * cross-page stitching — callers that need every physical page should
+ * use `pages: true` instead. No new response field. */
+ pageMarkers: z.boolean().optional(),
// Experimental: route this request through the fire-pdf async pipeline
// (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
// to sync on any async-path failure, so user-visible behavior is unchanged
@@ -583,6 +590,16 @@ export function getPDFBlocks(parsers?: Parsers): boolean {
return false;
}
+export function getPDFPageMarkers(parsers?: Parsers): boolean {
+ if (!parsers) return false;
+ for (const parser of parsers) {
+ if (typeof parser === "object" && parser.type === "pdf") {
+ return parser.pageMarkers === true;
+ }
+ }
+ return false;
+}
+
export function getFirePdfAsync(parsers?: Parsers): boolean {
if (!parsers) return false;
for (const parser of parsers) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 315e68edac..0401806337 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -30,6 +30,7 @@ import { hasFormatOfType } from "../../../lib/format-utils";
import {
getPDFBlocks,
getPDFPageMarkdown,
+ getPDFPageMarkers,
} from "../../../controllers/v2/types";
import type { PdfMetadata, PdfPageBlocks } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
@@ -567,9 +568,10 @@ export function shouldUseIndex(meta: Meta) {
!hasFormatOfType(meta.options.formats, "changeTracking") &&
!hasFormatOfType(meta.options.formats, "branding") &&
// The URL index does not yet persist physical-page or typed-block
- // capability metadata.
+ // capability metadata, and its markdown never carries page markers.
!getPDFPageMarkdown(meta.options.parsers) &&
!getPDFBlocks(meta.options.parsers) &&
+ !getPDFPageMarkers(meta.options.parsers) &&
!hasCustomScreenshotSettings &&
meta.options.maxAge !== 0 &&
(meta.options.headers === undefined ||
@@ -680,17 +682,19 @@ export async function buildFallbackList(meta: Meta): Promise<
_engines.push(...indexEngines);
meta.internalOptions.forceEngine = indexEngines;
} else if (meta.internalOptions.agentIndexOnly) {
- // Index documents carry no physical-page or typed-block payloads, so an
- // index-only request that demands them can only be answered wrong. Fail
- // loud with the canonical index-only error (maps to a clean 4xx and
- // tells the caller how to unlock live scraping) instead of silently
- // serving a document without the capability.
+ // Index documents carry no physical-page or typed-block payloads, and
+ // their markdown never carries page markers, so an index-only request
+ // that demands them can only be answered wrong. Fail loud with the
+ // canonical index-only error (maps to a clean 4xx and tells the caller
+ // how to unlock live scraping) instead of silently serving a document
+ // without the capability.
if (
getPDFPageMarkdown(meta.options.parsers) ||
- getPDFBlocks(meta.options.parsers)
+ getPDFBlocks(meta.options.parsers) ||
+ getPDFPageMarkers(meta.options.parsers)
) {
meta.logger.warn(
- "agentIndexOnly request demands pageMarkdown/blocks, which the URL index cannot serve",
+ "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve",
{ parsers: meta.options.parsers },
);
throw new AgentIndexOnlyError();
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index e2f9596b41..223a297c8c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -40,6 +40,7 @@ import {
getPDFBlocks,
getPDFMaxPages,
getPDFPageMarkdown,
+ getPDFPageMarkers,
shouldParsePDF,
} from "../../../../controllers/v2/types";
import { hasFormatOfType } from "../../../../lib/format-utils";
@@ -64,8 +65,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
// Page-aware and block-aware results are capability-specific and are not
// represented in the URL index schema yet. Do not write an entry that
// could later be served without its required pages/blocks payload.
+ // Marker-bearing markdown is mutated output — never index it either.
!getPDFPageMarkdown(meta.options.parsers) &&
!getPDFBlocks(meta.options.parsers) &&
+ !getPDFPageMarkers(meta.options.parsers) &&
!meta.options.parsers?.some(parser => {
if (
typeof parser === "object" &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index 6ad00666c8..4acd9f5045 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -211,3 +211,105 @@ describe("scrapePDFWithFirePDF typed blocks", () => {
).rejects.toThrow(/did not include requested typed blocks/);
});
});
+
+describe("scrapePDFWithFirePDF page markers", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("requests marker-joined markdown and returns it verbatim", async () => {
+ const marked = "Page 1\n\n---\n\n\n\nPage 2";
+ mockedRobustFetch.mockResolvedValue({
+ markdown: marked,
+ failed_pages: null,
+ pages_processed: 2,
+ page_markers: true,
+ } as any);
+
+ const result = await scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ false,
+ false,
+ true,
+ );
+
+ expect(mockedRobustFetch).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({ page_markers: true }),
+ }),
+ );
+ expect(result.markdown).toBe(marked);
+ });
+
+ it("does not send page_markers for plain requests", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "plain",
+ failed_pages: null,
+ pages_processed: 1,
+ } as any);
+
+ await scrapePDFWithFirePDF(makeMeta(), "BASE64");
+
+ expect(
+ ((mockedRobustFetch.mock.calls[0][0] as any).body as any).page_markers,
+ ).toBeUndefined();
+ });
+
+ it("rejects FirePDF responses that do not acknowledge page markers", async () => {
+ // An old fire-pdf build ignores the unknown `page_markers` request field
+ // and returns ordinary markdown with no echo — indistinguishable from
+ // marked output by content alone, so the missing echo must fail loud.
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Page 1\n\n---\n\nPage 2",
+ failed_pages: null,
+ pages_processed: 2,
+ } as any);
+
+ await expect(
+ scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ false,
+ false,
+ true,
+ ),
+ ).rejects.toThrow(/did not acknowledge requested page markers/);
+ });
+
+ it("composes page_markers with include_blocks on the wire", async () => {
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Page 1\n\n---\n\n\n\nPage 2",
+ failed_pages: null,
+ pages_processed: 2,
+ blocks: [],
+ page_markers: true,
+ } as any);
+
+ await scrapePDFWithFirePDF(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ false,
+ true,
+ true,
+ );
+
+ expect(mockedRobustFetch).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({
+ include_blocks: true,
+ page_markers: true,
+ }),
+ }),
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 842d0d0358..03d351129f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -348,6 +348,99 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(error.reason).toBe("http_5xx");
});
+ it("requests page markers and returns the acknowledged marked markdown", async () => {
+ const marked = "Page 1\n\n---\n\n\n\nPage 2";
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: marked,
+ pages_processed: 2,
+ page_markers: true,
+ },
+ },
+ },
+ ]);
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ { fetchImpl, sleepImpl: noopSleep },
+ false,
+ false,
+ true,
+ );
+
+ expect((calls[0].body as any).options.pageMarkers).toBe(true);
+ expect(result.markdown).toBe(marked);
+ });
+
+ it("fails a marker request when the result lacks the page_markers echo", async () => {
+ // An older worker ignores the unknown pageMarkers option and persists
+ // ordinary markdown; without the echo this must fail (and fall back to
+ // the sync path) rather than cache unmarked markdown as marked output.
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ lane: "fast",
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "Page 1\n\n---\n\nPage 2",
+ pages_processed: 2,
+ },
+ },
+ },
+ ]);
+
+ const error = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ "auto",
+ { fetchImpl, sleepImpl: noopSleep },
+ false,
+ false,
+ true,
+ ).catch(error => error);
+
+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(error.reason).toBe("http_5xx");
+ });
+
it("requests and returns typed blocks, tolerating the legacy pages alias", async () => {
const blocks = [
{
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index d697b38317..24c88fcd7e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -82,6 +82,43 @@ describe("FirePDF page-markdown cache capabilities", () => {
expect(cacheKeyShape("auto", 5, false, true).cacheable).toBe(false);
});
+ it("maps page-marker requests into a fully disjoint variant family", () => {
+ // pageMarkers rewrites the markdown itself, so no marker lookup may ever
+ // name a non-marker variant (and vice versa — see the plain-request test:
+ // its probe list is unchanged and marker-free).
+ expect(cacheKeyShape("auto", undefined, false, false, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "markers-v1",
+ baseVariant: "markers-v1",
+ lookupVariants: [
+ "markers-v1",
+ "page-markdown-markers-v1",
+ "ocr-markers-v1",
+ "ocr-page-markdown-markers-v1",
+ ],
+ });
+ expect(cacheKeyShape("ocr", undefined, false, false, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "ocr-markers-v1",
+ lookupVariants: ["ocr-markers-v1", "ocr-page-markdown-markers-v1"],
+ });
+ expect(cacheKeyShape("auto", undefined, false, true, true)).toMatchObject({
+ cacheable: true,
+ ownVariant: "blocks-markers-v1",
+ baseVariant: "markers-v1",
+ lookupVariants: [
+ "blocks-markers-v1",
+ "page-markdown-blocks-markers-v1",
+ "ocr-blocks-markers-v1",
+ "ocr-page-markdown-blocks-markers-v1",
+ ],
+ });
+ expect(cacheKeyShape("fast", undefined, false, false, true).cacheable).toBe(
+ false,
+ );
+ expect(cacheKeyShape("auto", 5, false, false, true).cacheable).toBe(false);
+ });
+
it("keeps the historical probe list for plain requests", () => {
expect(cacheKeyShape("auto", undefined, false, false)).toMatchObject({
cacheable: true,
@@ -504,4 +541,73 @@ describe("FirePDF page-markdown cache capabilities", () => {
undefined,
);
});
+
+ it("saves marker results only under marker variants (never the base key)", async () => {
+ const marked = "Page 1\n\n---\n\n\n\nPage 2";
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ pageMarkers: true,
+ result: {
+ markdown: marked,
+ html: "
marked
",
+ pagesProcessed: 2,
+ },
+ });
+
+ // Marker markdown must never back-fill the non-marker base key: a later
+ // plain request would silently receive marker-mutated markdown.
+ expect(saveCached).toHaveBeenCalledOnce();
+ expect(saveCached).toHaveBeenCalledWith(
+ "BASE64",
+ expect.objectContaining({ markdown: marked }),
+ "firepdf",
+ "markers-v1",
+ );
+ });
+
+ it("back-fills enriched marker results within the marker family only", async () => {
+ const marked = "Page 1\n\n---\n\n\n\nPage 2";
+ const blocks = [
+ { page: 1, width: 800, height: 1100, status: "ok", items: [] },
+ ];
+
+ await maybeSaveResult({
+ meta: makeMeta(),
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: true,
+ pageMarkers: true,
+ result: {
+ markdown: marked,
+ html: "marked
",
+ pagesProcessed: 2,
+ blocks,
+ },
+ });
+
+ expect(getCached).toHaveBeenCalledWith("BASE64", "firepdf", "markers-v1");
+ expect(saveCached).toHaveBeenCalledTimes(2);
+ expect(saveCached).toHaveBeenNthCalledWith(
+ 1,
+ "BASE64",
+ expect.objectContaining({ blocks }),
+ "firepdf",
+ "blocks-markers-v1",
+ );
+ expect(saveCached).toHaveBeenNthCalledWith(
+ 2,
+ "BASE64",
+ expect.not.objectContaining({ blocks: expect.anything() }),
+ "firepdf",
+ "markers-v1",
+ );
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
index 5a12a42040..6e80a2c5f0 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts
@@ -52,6 +52,15 @@ describe("PDF page-markdown URL index policy", () => {
},
}),
).toBe(false);
+ expect(
+ shouldUseIndex({
+ ...baseMeta,
+ options: {
+ ...baseMeta.options,
+ parsers: [{ type: "pdf", pageMarkers: true }],
+ },
+ }),
+ ).toBe(false);
} finally {
(
config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean }
@@ -116,4 +125,31 @@ describe("PDF page-markdown URL index policy", () => {
expect(result).toBe(document);
expect(result.metadata.indexId).toBeUndefined();
});
+
+ it("does not write marker-bearing markdown to the URL index", async () => {
+ const document = {
+ markdown: "Page 1\n\n---\n\n\n\nPage 2",
+ rawHtml: "whole document
",
+ metadata: {
+ sourceURL: "https://example.com/file.pdf",
+ },
+ } as any;
+ const meta = {
+ url: "https://example.com/file.pdf",
+ winnerEngine: "pdf",
+ options: {
+ storeInCache: true,
+ parsers: [{ type: "pdf", pageMarkers: true }],
+ },
+ internalOptions: {
+ isParse: false,
+ zeroDataRetention: false,
+ },
+ } as any;
+
+ const result = await sendDocumentToIndex(meta, document);
+
+ expect(result).toBe(document);
+ expect(result.metadata.indexId).toBeUndefined();
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 3174dc88ca..53e135bd6e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -39,6 +39,7 @@ export async function scrapePDFWithFirePDFAsync(
deps: FirePdfAsyncDeps = {},
includePageMarkdown = false,
includeBlocks = false,
+ pageMarkers = false,
): Promise {
const fetchImpl = deps.fetchImpl ?? undiciFetch;
const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;
@@ -57,6 +58,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
@@ -68,6 +70,7 @@ export async function scrapePDFWithFirePDFAsync(
pagesProcessed,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
if (cached) return cached;
@@ -93,6 +96,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
@@ -127,6 +131,7 @@ export async function scrapePDFWithFirePDFAsync(
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
deadlineAt,
teamConcurrency,
fetchImpl,
@@ -192,6 +197,16 @@ export async function scrapePDFWithFirePDFAsync(
note: "FirePDF result omitted requested typed blocks",
});
}
+ if (pageMarkers && fetched.page_markers !== true) {
+ // Markers are baked into the markdown, so the missing echo is the only
+ // signal the worker build ignored the option; accepting the result
+ // would cache unmarked markdown under a marker cache variant. Fail the
+ // async attempt — the caller retries synchronously, where the same
+ // echo contract applies.
+ failAsync(meta, "http_5xx", {
+ note: "FirePDF result did not acknowledge requested page markers",
+ });
+ }
const durationMs = now() - overallStartedAt;
firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);
@@ -222,6 +237,7 @@ export async function scrapePDFWithFirePDFAsync(
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
result: processorResult,
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index 647aaa3ae8..d62fc723e9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -18,6 +18,19 @@ const OCR_BLOCKS_VARIANT = "ocr-blocks-v1";
const PAGE_MARKDOWN_BLOCKS_VARIANT = "page-markdown-blocks-v1";
const OCR_PAGE_MARKDOWN_BLOCKS_VARIANT = "ocr-page-markdown-blocks-v1";
+// `page_markers` rewrites the document markdown itself (inter-page
+// `` separators), unlike pages/blocks which are extra
+// payloads beside unchanged markdown. Marker and non-marker artifacts can
+// therefore never serve each other. Follow the `mode: ocr` dedicated-variant
+// precedent: map every variant name into a disjoint `…markers…` family.
+// Within that family the ocr/pages/blocks capability lattice applies
+// unchanged, because those artifacts differ only in sidecars again.
+function withPageMarkers(variant: string | undefined): string {
+ if (variant === undefined) return "markers-v1";
+ if (variant === "ocr") return "ocr-markers-v1";
+ return variant.replace(/-v1$/, "-markers-v1");
+}
+
function isValidCachedDocument(
value: unknown,
): value is Pick & { markdown: string } {
@@ -67,6 +80,7 @@ export function cacheKeyShape(
maxPages: number | undefined,
includePageMarkdown: boolean,
includeBlocks: boolean,
+ pageMarkers = false,
) {
const cacheable = mode !== "fast" && !maxPages;
const isOcr = mode === "ocr";
@@ -116,6 +130,14 @@ export function cacheKeyShape(
: isOcr
? ["ocr", OCR_PAGE_MARKDOWN_VARIANT]
: [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT];
+ if (pageMarkers) {
+ return {
+ cacheable,
+ ownVariant: withPageMarkers(ownVariant),
+ baseVariant: withPageMarkers(baseVariant),
+ lookupVariants: lookupVariants.map(withPageMarkers),
+ };
+ }
return { cacheable, ownVariant, baseVariant, lookupVariants };
}
@@ -127,6 +149,7 @@ export async function tryGetCached(
pagesProcessed: number | undefined,
includePageMarkdown: boolean,
includeBlocks: boolean,
+ pageMarkers = false,
): Promise {
if (meta.internalOptions.zeroDataRetention) return null;
const { cacheable, lookupVariants } = cacheKeyShape(
@@ -134,6 +157,7 @@ export async function tryGetCached(
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
if (!cacheable) return null;
@@ -186,6 +210,7 @@ export async function maybeSaveResult(args: {
maxPages: number | undefined;
includePageMarkdown: boolean;
includeBlocks: boolean;
+ pageMarkers?: boolean;
result: PDFProcessorResult & { markdown: string };
}): Promise {
const {
@@ -195,6 +220,7 @@ export async function maybeSaveResult(args: {
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers = false,
result,
} = args;
if (meta.internalOptions.zeroDataRetention) return;
@@ -203,6 +229,7 @@ export async function maybeSaveResult(args: {
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
if (!cacheable) return;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index 47f197e199..7e5bbf7341 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -125,6 +125,11 @@ export const resultResponseSchema = z.object({
pages_processed: z.number().optional(),
failed_pages: z.array(z.number()).nullable().optional(),
partial_pages: z.array(z.number()).nullable().optional(),
+ // Echo of an honored pageMarkers job option. Markers are baked into
+ // `markdown` and their absence is not detectable by content, so the echo
+ // is the only proof the fire-pdf worker build understood the option —
+ // older workers ignore unknown option keys and omit it.
+ page_markers: z.literal(true).optional(),
});
export type PollResponse = z.infer;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 968e0ad254..716cc47915 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -21,6 +21,7 @@ type SubmitArgs = {
mode: PDFMode | undefined;
includePageMarkdown: boolean;
includeBlocks: boolean;
+ pageMarkers: boolean;
deadlineAt: string;
/** Team's sold concurrency from the ACUC (ENG-5049 account context).
* Optional: entitlement lookup must never block or fail a scrape. */
@@ -62,6 +63,7 @@ export async function submitJob(args: SubmitArgs): Promise {
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
deadlineAt,
teamConcurrency,
fetchImpl,
@@ -92,6 +94,12 @@ export async function submitJob(args: SubmitArgs): Promise {
...(mode !== undefined && { mode }),
...(includePageMarkdown && { include_page_markdown: true }),
...(includeBlocks && { include_blocks: true }),
+ // Intentionally camelCase, unlike its siblings: the fire-pdf async
+ // /jobs options schema named this key `pageMarkers` (fire-pdf
+ // api/src/http/schemas/jobs.ts) while the sync /ocr path uses
+ // `page_markers`. Sending snake_case here would be rejected as an
+ // unknown option.
+ ...(pageMarkers && { pageMarkers: true }),
},
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 3d9b11311c..de9d45779a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -47,6 +47,7 @@ export async function scrapePDFWithFirePDF(
mode?: PDFMode,
includePageMarkdown = false,
includeBlocks = false,
+ pageMarkers = false,
): Promise {
const logger = meta.logger;
@@ -62,6 +63,11 @@ export async function scrapePDFWithFirePDF(
// running fire-pdf again.
// - `fast` is bypassed entirely (hard cost ceiling — must fail on
// scanned PDFs, not serve a cached OCR result).
+ // - `page_markers` rewrites the document markdown itself (inter-page
+ // `` separators), so marker requests read/write a
+ // fully disjoint `…markers…` variant family — a base-variant entry
+ // must never be served for a marker request and vice versa. See
+ // cacheKeyShape in fire-pdf/cache.ts.
const cacheable =
mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;
const cached = cacheable
@@ -73,6 +79,7 @@ export async function scrapePDFWithFirePDF(
pagesProcessed,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
)
: null;
if (cached) return cached;
@@ -124,6 +131,7 @@ export async function scrapePDFWithFirePDF(
...(mode !== undefined && { mode }),
...(includePageMarkdown && { include_page_markdown: true }),
...(includeBlocks && { include_blocks: true }),
+ ...(pageMarkers && { page_markers: true }),
// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
// these as optional — older fire-pdf builds will ignore unknown fields.
team_id: meta.internalOptions.teamId,
@@ -143,6 +151,12 @@ export async function scrapePDFWithFirePDF(
pages_processed: z.number().optional(),
pages: firePdfPagesSchema,
blocks: firePdfBlocksSchema,
+ // Echo of an honored page_markers request. Markers are baked into
+ // `markdown` and their absence is not reliably detectable there (a
+ // single-page or fully-stitched document legitimately has none), so
+ // the echo is the only proof the fire-pdf build understood the
+ // option — older builds ignore unknown request fields and omit it.
+ page_markers: z.literal(true).optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
@@ -157,6 +171,13 @@ export async function scrapePDFWithFirePDF(
if (includeBlocks && resp.blocks === undefined) {
throw new Error("FirePDF response did not include requested typed blocks");
}
+ if (pageMarkers && resp.page_markers !== true) {
+ // Without the echo, the markdown is ordinary unmarked output; caching
+ // it under a marker variant would silently poison the marker cache.
+ throw new Error(
+ "FirePDF response did not acknowledge requested page markers",
+ );
+ }
const pages = resp.pages_processed ?? pagesProcessed;
logger.info("FirePDF completed", {
@@ -185,6 +206,7 @@ export async function scrapePDFWithFirePDF(
maxPages,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
result: processorResult,
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index e40c2ae194..bc67579ead 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -20,6 +20,7 @@ import {
getPDFMode,
getPDFPageMarkdown,
getPDFBlocks,
+ getPDFPageMarkers,
getFirePdfAsync,
} from "../../../../controllers/v2/types";
import type { PDFMode } from "../../../../controllers/v2/types";
@@ -64,6 +65,7 @@ export async function scrapePDF(meta: Meta): Promise {
const mode: PDFMode = getPDFMode(meta.options.parsers);
const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers);
const includeBlocks = getPDFBlocks(meta.options.parsers);
+ const pageMarkers = getPDFPageMarkers(meta.options.parsers);
if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) {
throw new Error(
@@ -77,6 +79,12 @@ export async function scrapePDF(meta: Meta): Promise {
);
}
+ if (pageMarkers && !config.FIRE_PDF_BASE_URL) {
+ throw new Error(
+ "Page markers are unavailable because FirePDF is not configured",
+ );
+ }
+
if (!shouldParse) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
const content = (await readFile(meta.pdfPrefetch.filePath)).toString(
@@ -187,7 +195,10 @@ export async function scrapePDF(meta: Meta): Promise {
let shadowPagesNeedingOcr: number[] | undefined;
const forceFirePDF =
- (!!meta.options.__forceFirePDF || includePageMarkdown || includeBlocks) &&
+ (!!meta.options.__forceFirePDF ||
+ includePageMarkdown ||
+ includeBlocks ||
+ pageMarkers) &&
!!config.FIRE_PDF_BASE_URL;
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
@@ -471,17 +482,18 @@ export async function scrapePDF(meta: Meta): Promise {
undefined,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
} catch (error) {
if (
- (!includePageMarkdown && !includeBlocks) ||
+ (!includePageMarkdown && !includeBlocks && !pageMarkers) ||
error instanceof RemoveFeatureError ||
error instanceof AbortManagerThrownError
) {
throw error;
}
meta.logger.warn(
- "FirePDF async page markdown/blocks failed -- retrying synchronously",
+ "FirePDF async page markdown/blocks/markers failed -- retrying synchronously",
{
method: "scrapePDF/firePDFFallback",
error,
@@ -502,6 +514,7 @@ export async function scrapePDF(meta: Meta): Promise {
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
} else {
@@ -513,6 +526,7 @@ export async function scrapePDF(meta: Meta): Promise {
mode,
includePageMarkdown,
includeBlocks,
+ pageMarkers,
);
}
effectivePageCount = reconcilePageCountWithFirePdf(
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
index ec8e9c71ec..612fbe99c9 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
@@ -426,6 +426,25 @@ public void CrawlJob_DeserializesCorrectly()
Assert.Equal(2, job.Data.Count);
}
+ [Fact]
+ public void PdfParser_SerializesPageMarkers()
+ {
+ var parser = new PdfParser
+ {
+ Mode = "auto",
+ Pages = true,
+ Blocks = true,
+ PageMarkers = true
+ };
+
+ var json = JsonSerializer.Serialize(parser, JsonOptions);
+ Assert.Contains("\"type\":\"pdf\"", json);
+ Assert.Contains("\"mode\":\"auto\"", json);
+ Assert.Contains("\"pages\":true", json);
+ Assert.Contains("\"blocks\":true", json);
+ Assert.Contains("\"pageMarkers\":true", json);
+ }
+
[Fact]
public void JsonFormat_HasCorrectType()
{
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index a1da003731..7bc41f43a8 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
firecrawl-sdk
- 1.12.0
+ 1.13.0
Firecrawl
Firecrawl
.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction
diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
index e3bcb1a65e..169db89f80 100644
--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
@@ -714,7 +714,7 @@ private async Task PaginateMonitorCheckAsync(
// INTERNAL UTILITIES
// ================================================================
- private const string SdkOrigin = "dotnet-sdk@1.12.0";
+ private const string SdkOrigin = "dotnet-sdk@1.13.0";
private static Dictionary BuildBody(object? options)
{
diff --git a/apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs b/apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs
new file mode 100644
index 0000000000..f8f393c78e
--- /dev/null
+++ b/apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs
@@ -0,0 +1,45 @@
+using System.Text.Json.Serialization;
+
+namespace Firecrawl.Models;
+
+///
+/// PDF parser configuration for use in /
+/// .
+///
+public class PdfParser
+{
+ [JsonPropertyName("type")]
+ public string Type { get; } = "pdf";
+
+ [JsonPropertyName("mode")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public string? Mode { get; set; }
+
+ [JsonPropertyName("maxPages")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public int? MaxPages { get; set; }
+
+ ///
+ /// Include physical per-page markdown. Populates document.pages.
+ ///
+ [JsonPropertyName("pages")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public bool? Pages { get; set; }
+
+ ///
+ /// Include per-page typed layout blocks. Populates document.blocks.
+ ///
+ [JsonPropertyName("blocks")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public bool? Blocks { get; set; }
+
+ ///
+ /// Join PDF pages in document.markdown with
+ /// \n\n---\n\n<!-- page N -->\n\n. Markers appear between
+ /// pages only; numbering may skip pages merged by cross-page stitching —
+ /// use when every physical page is needed.
+ ///
+ [JsonPropertyName("pageMarkers")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public bool? PageMarkers { get; set; }
+}
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index be40771720..f65ff09189 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -37,6 +37,11 @@ type PDFParser struct {
MaxPages *int `json:"maxPages,omitempty"`
Pages *bool `json:"pages,omitempty"`
Blocks *bool `json:"blocks,omitempty"`
+ // PageMarkers joins PDF pages in document.markdown with
+ // `\n\n---\n\n\n\n`. Markers appear between pages only;
+ // numbering may skip pages merged by cross-page stitching — use Pages
+ // when every physical page is needed. No new response field.
+ PageMarkers *bool `json:"pageMarkers,omitempty"`
}
// PdfPage is physical markdown for a single PDF page.
@@ -76,12 +81,12 @@ type PdfPageBlocks struct {
// ProductProfile represents structured product data extracted from a page
// via the `product` scrape format.
type ProductProfile struct {
- Title string `json:"title"`
- Brand string `json:"brand,omitempty"`
- Category string `json:"category,omitempty"`
- URL string `json:"url"`
- Description string `json:"description,omitempty"`
- Variants []ProductVariant `json:"variants,omitempty"`
+ Title string `json:"title"`
+ Brand string `json:"brand,omitempty"`
+ Category string `json:"category,omitempty"`
+ URL string `json:"url"`
+ Description string `json:"description,omitempty"`
+ Variants []ProductVariant `json:"variants,omitempty"`
}
// ProductImage is a single product image.
diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go
index 4f8db40137..40c80df091 100644
--- a/apps/go-sdk/options_test.go
+++ b/apps/go-sdk/options_test.go
@@ -67,16 +67,17 @@ func TestScrapeOptionsPreservesStringFormats(t *testing.T) {
func TestScrapeOptionsSerializesPDFParserPagesAndBlocks(t *testing.T) {
pages := true
blocks := true
+ pageMarkers := true
payload, err := json.Marshal(ScrapeOptions{
Parsers: []interface{}{
- PDFParser{Type: "pdf", Mode: "auto", Pages: &pages, Blocks: &blocks},
+ PDFParser{Type: "pdf", Mode: "auto", Pages: &pages, Blocks: &blocks, PageMarkers: &pageMarkers},
},
})
if err != nil {
t.Fatalf("Marshal ScrapeOptions: %v", err)
}
- want := `"parsers":[{"type":"pdf","mode":"auto","pages":true,"blocks":true}]`
+ want := `"parsers":[{"type":"pdf","mode":"auto","pages":true,"blocks":true,"pageMarkers":true}]`
if !strings.Contains(string(payload), want) {
t.Fatalf("serialized parsers = %s, want to contain %s", payload, want)
}
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index c27aa29280..186497366a 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.11.0"
+const Version = "1.12.0"
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index e9c98c03a6..3d268db2a8 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.14.0"
+version = "1.15.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
index f0c9f8ada7..3b725f3b36 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
@@ -37,7 +37,7 @@
public class FirecrawlClient {
private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";
- private static final String SDK_ORIGIN = "java-sdk@1.14.0";
+ private static final String SDK_ORIGIN = "java-sdk@1.15.0";
private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutes
private static final int DEFAULT_MAX_RETRIES = 3;
private static final double DEFAULT_BACKOFF_FACTOR = 0.5;
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
index 01b133df1c..ff88c6f5fa 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
@@ -127,6 +127,7 @@ private Builder() {}
public Builder excludeTags(List excludeTags) { this.excludeTags = excludeTags; return this; }
public Builder onlyMainContent(Boolean onlyMainContent) { this.onlyMainContent = onlyMainContent; return this; }
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
+ /** Parsers to use (e.g., "pdf" or PdfParser with maxPages, pages, blocks, pageMarkers). */
public Builder parsers(List