From 24b8112640049504ed4c85fdd0fa717e221622ec Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:26:21 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(v2):=20pageMarkers=20PDF=20parser=20op?= =?UTF-8?q?tion=20=E2=80=94=20per-page=20page=20attribution=20in=20documen?= =?UTF-8?q?t.markdown=20(#4365)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(v2): pageMarkers PDF parser option — per-page attribution in document.markdown Adds `parsers: [{ type: "pdf", pageMarkers: true }]` (default false) to v2 /parse and /scrape. When set, PDF pages in document.markdown are joined with `\n\n---\n\n\n\n` where N is the 1-based physical page of the content that follows. No new response field. Semantics: markers appear between pages only (no leading marker for page 1); numbering may skip pages merged by cross-page stitching — callers that need every physical page should use `pages: true`. Composes with pages, blocks, maxPages, and mode. Plumbing follows the pages/blocks precedent (#4347): - schema field flows through the pageMarkdown-alias transform + getPDFPageMarkers() accessor - firePDF sync engine sends page_markers; async /jobs submit sends pageMarkers; both force the fire-pdf engine (no silent MinerU fallback without markers) - excluded from URL-index reads/writes and agentIndexOnly (index markdown never carries markers) - PDF cache: pageMarkers mutates markdown itself (unlike the pages/blocks sidecars), so marker requests read/write a fully disjoint …markers… variant family — a base-variant entry is never served for a marker request and vice versa (mode:ocr precedent) - JS/Python SDK parser types updated Requires fire-pdf's page_markers support on the sync /ocr path to be deployed first (older fire-pdf builds ignore unknown fields). Co-Authored-By: Claude Fable 5 * review: fail loud on missing page_markers echo, drop SDK changes, fix fallback log - Revert JS/Python SDK changes — this PR is API-only for now; SDK support lands separately. - Validate the new fire-pdf page_markers response echo (firecrawl/fire-pdf#629): markers are baked into markdown and their absence is not detectable by content, so a build that ignores the unknown request field must fail loud instead of silently caching unmarked markdown under marker cache variants. - Document why the async /jobs option is camelCase pageMarkers: that is the field fire-pdf's jobs schema defines, and its handler 400s unknown option keys, so snake_case would be rejected. - Mention markers in the async→sync fallback warn message. Co-Authored-By: Claude Fable 5 * review: validate the page_markers echo on the async /jobs result too The sync path validates fire-pdf's page_markers acknowledgment, but the async path cached results with none — an api/worker version skew (submit accepts the pageMarkers option, an older worker silently ignores the stored key) would persist unmarked markdown under marker cache variants. fire-pdf now echoes page_markers on the /jobs result JSON (firecrawl/fire-pdf#629); reject marker results lacking the echo and fall back to the sync path, where the same contract applies. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../src/__tests__/pdf-parser-options.test.ts | 22 ++++ apps/api/src/controllers/v2/types.ts | 17 +++ .../src/scraper/scrapeURL/engines/index.ts | 20 ++-- .../scraper/scrapeURL/engines/index/index.ts | 3 + .../engines/pdf/__tests__/firePDF.test.ts | 102 +++++++++++++++++ .../pdf/__tests__/firePDFAsync.test.ts | 93 +++++++++++++++ .../pdf/__tests__/firePDFCache.test.ts | 106 ++++++++++++++++++ .../pdf/__tests__/indexPageMarkdown.test.ts | 36 ++++++ .../scrapeURL/engines/pdf/fire-pdf/async.ts | 16 +++ .../scrapeURL/engines/pdf/fire-pdf/cache.ts | 27 +++++ .../scrapeURL/engines/pdf/fire-pdf/schema.ts | 5 + .../scrapeURL/engines/pdf/fire-pdf/submit.ts | 8 ++ .../scraper/scrapeURL/engines/pdf/firePDF.ts | 22 ++++ .../scraper/scrapeURL/engines/pdf/index.ts | 20 +++- 14 files changed, 486 insertions(+), 11 deletions(-) diff --git a/apps/api/src/__tests__/pdf-parser-options.test.ts b/apps/api/src/__tests__/pdf-parser-options.test.ts index f891efe6e9..d3f783f715 100644 --- a/apps/api/src/__tests__/pdf-parser-options.test.ts +++ b/apps/api/src/__tests__/pdf-parser-options.test.ts @@ -1,6 +1,7 @@ import { getPDFBlocks, getPDFPageMarkdown, + getPDFPageMarkers, scrapeOptions, } from "../controllers/v2/types"; @@ -23,6 +24,27 @@ describe("PDF parser option getters", () => { expect(getPDFBlocks([{ type: "pdf" }])).toBe(false); expect(getPDFBlocks(undefined)).toBe(false); }); + + it("reads the `pageMarkers` option", () => { + expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: true }])).toBe(true); + expect(getPDFPageMarkers([{ type: "pdf", pageMarkers: false }])).toBe( + false, + ); + expect(getPDFPageMarkers([{ type: "pdf" }])).toBe(false); + expect(getPDFPageMarkers(["pdf"])).toBe(false); + expect(getPDFPageMarkers(undefined)).toBe(false); + }); + + it("keeps `pageMarkers` flowing through the alias-normalizing transform", () => { + const parser = parsePdfParser({ + type: "pdf", + pageMarkers: true, + pageMarkdown: true, + }); + expect(parser.pageMarkers).toBe(true); + expect(parser.pages).toBe(true); + expect(getPDFPageMarkers([parser as any])).toBe(true); + }); }); describe("deprecated pageMarkdown alias", () => { diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index 54233c7991..496d00384f 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -497,6 +497,13 @@ const pdfParserWithOptions = z * reading order) alongside document markdown — populates * `document.blocks`. */ blocks: z.boolean().optional(), + /** Join PDF pages in `document.markdown` with + * `\n\n---\n\n\n\n` where N is the 1-based physical page + * of the content that follows. Markers appear between pages only (no + * leading marker for page 1), and numbering may skip pages merged by + * cross-page stitching — callers that need every physical page should + * use `pages: true` instead. No new response field. */ + pageMarkers: z.boolean().optional(), // Experimental: route this request through the fire-pdf async pipeline // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back // to sync on any async-path failure, so user-visible behavior is unchanged @@ -583,6 +590,16 @@ export function getPDFBlocks(parsers?: Parsers): boolean { return false; } +export function getPDFPageMarkers(parsers?: Parsers): boolean { + if (!parsers) return false; + for (const parser of parsers) { + if (typeof parser === "object" && parser.type === "pdf") { + return parser.pageMarkers === true; + } + } + return false; +} + export function getFirePdfAsync(parsers?: Parsers): boolean { if (!parsers) return false; for (const parser of parsers) { diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts index 315e68edac..0401806337 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index.ts @@ -30,6 +30,7 @@ import { hasFormatOfType } from "../../../lib/format-utils"; import { getPDFBlocks, getPDFPageMarkdown, + getPDFPageMarkers, } from "../../../controllers/v2/types"; import type { PdfMetadata, PdfPageBlocks } from "./pdf/types"; import { BrandingProfile } from "../../../types/branding"; @@ -567,9 +568,10 @@ export function shouldUseIndex(meta: Meta) { !hasFormatOfType(meta.options.formats, "changeTracking") && !hasFormatOfType(meta.options.formats, "branding") && // The URL index does not yet persist physical-page or typed-block - // capability metadata. + // capability metadata, and its markdown never carries page markers. !getPDFPageMarkdown(meta.options.parsers) && !getPDFBlocks(meta.options.parsers) && + !getPDFPageMarkers(meta.options.parsers) && !hasCustomScreenshotSettings && meta.options.maxAge !== 0 && (meta.options.headers === undefined || @@ -680,17 +682,19 @@ export async function buildFallbackList(meta: Meta): Promise< _engines.push(...indexEngines); meta.internalOptions.forceEngine = indexEngines; } else if (meta.internalOptions.agentIndexOnly) { - // Index documents carry no physical-page or typed-block payloads, so an - // index-only request that demands them can only be answered wrong. Fail - // loud with the canonical index-only error (maps to a clean 4xx and - // tells the caller how to unlock live scraping) instead of silently - // serving a document without the capability. + // Index documents carry no physical-page or typed-block payloads, and + // their markdown never carries page markers, so an index-only request + // that demands them can only be answered wrong. Fail loud with the + // canonical index-only error (maps to a clean 4xx and tells the caller + // how to unlock live scraping) instead of silently serving a document + // without the capability. if ( getPDFPageMarkdown(meta.options.parsers) || - getPDFBlocks(meta.options.parsers) + getPDFBlocks(meta.options.parsers) || + getPDFPageMarkers(meta.options.parsers) ) { meta.logger.warn( - "agentIndexOnly request demands pageMarkdown/blocks, which the URL index cannot serve", + "agentIndexOnly request demands pageMarkdown/blocks/pageMarkers, which the URL index cannot serve", { parsers: meta.options.parsers }, ); throw new AgentIndexOnlyError(); diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index e2f9596b41..223a297c8c 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -40,6 +40,7 @@ import { getPDFBlocks, getPDFMaxPages, getPDFPageMarkdown, + getPDFPageMarkers, shouldParsePDF, } from "../../../../controllers/v2/types"; import { hasFormatOfType } from "../../../../lib/format-utils"; @@ -64,8 +65,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { // Page-aware and block-aware results are capability-specific and are not // represented in the URL index schema yet. Do not write an entry that // could later be served without its required pages/blocks payload. + // Marker-bearing markdown is mutated output — never index it either. !getPDFPageMarkdown(meta.options.parsers) && !getPDFBlocks(meta.options.parsers) && + !getPDFPageMarkers(meta.options.parsers) && !meta.options.parsers?.some(parser => { if ( typeof parser === "object" && diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts index 6ad00666c8..4acd9f5045 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts @@ -211,3 +211,105 @@ describe("scrapePDFWithFirePDF typed blocks", () => { ).rejects.toThrow(/did not include requested typed blocks/); }); }); + +describe("scrapePDFWithFirePDF page markers", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("requests marker-joined markdown and returns it verbatim", async () => { + const marked = "Page 1\n\n---\n\n\n\nPage 2"; + mockedRobustFetch.mockResolvedValue({ + markdown: marked, + failed_pages: null, + pages_processed: 2, + page_markers: true, + } as any); + + const result = await scrapePDFWithFirePDF( + makeMeta(), + "BASE64", + undefined, + undefined, + "auto", + false, + false, + true, + ); + + expect(mockedRobustFetch).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ page_markers: true }), + }), + ); + expect(result.markdown).toBe(marked); + }); + + it("does not send page_markers for plain requests", async () => { + mockedRobustFetch.mockResolvedValue({ + markdown: "plain", + failed_pages: null, + pages_processed: 1, + } as any); + + await scrapePDFWithFirePDF(makeMeta(), "BASE64"); + + expect( + ((mockedRobustFetch.mock.calls[0][0] as any).body as any).page_markers, + ).toBeUndefined(); + }); + + it("rejects FirePDF responses that do not acknowledge page markers", async () => { + // An old fire-pdf build ignores the unknown `page_markers` request field + // and returns ordinary markdown with no echo — indistinguishable from + // marked output by content alone, so the missing echo must fail loud. + mockedRobustFetch.mockResolvedValue({ + markdown: "Page 1\n\n---\n\nPage 2", + failed_pages: null, + pages_processed: 2, + } as any); + + await expect( + scrapePDFWithFirePDF( + makeMeta(), + "BASE64", + undefined, + undefined, + "auto", + false, + false, + true, + ), + ).rejects.toThrow(/did not acknowledge requested page markers/); + }); + + it("composes page_markers with include_blocks on the wire", async () => { + mockedRobustFetch.mockResolvedValue({ + markdown: "Page 1\n\n---\n\n\n\nPage 2", + failed_pages: null, + pages_processed: 2, + blocks: [], + page_markers: true, + } as any); + + await scrapePDFWithFirePDF( + makeMeta(), + "BASE64", + undefined, + undefined, + "auto", + false, + true, + true, + ); + + expect(mockedRobustFetch).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + include_blocks: true, + page_markers: true, + }), + }), + ); + }); +}); diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts index 842d0d0358..03d351129f 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts @@ -348,6 +348,99 @@ describe("scrapePDFWithFirePDFAsync", () => { expect(error.reason).toBe("http_5xx"); }); + it("requests page markers and returns the acknowledged marked markdown", async () => { + const marked = "Page 1\n\n---\n\n\n\nPage 2"; + const { fetchImpl, calls } = makeFetchFromSequence([ + { + matchUrl: /\/jobs$/, + matchMethod: "POST", + response: { + status: 200, + body: { + scrape_id: "scrape-id-test", + status: "done", + lane: "fast", + }, + }, + }, + { + matchUrl: /\/jobs\/scrape-id-test\/result$/, + matchMethod: "GET", + response: { + status: 200, + body: { + schema_version: 1, + markdown: marked, + pages_processed: 2, + page_markers: true, + }, + }, + }, + ]); + + const result = await scrapePDFWithFirePDFAsync( + makeMeta(), + "BASE64", + undefined, + undefined, + "auto", + { fetchImpl, sleepImpl: noopSleep }, + false, + false, + true, + ); + + expect((calls[0].body as any).options.pageMarkers).toBe(true); + expect(result.markdown).toBe(marked); + }); + + it("fails a marker request when the result lacks the page_markers echo", async () => { + // An older worker ignores the unknown pageMarkers option and persists + // ordinary markdown; without the echo this must fail (and fall back to + // the sync path) rather than cache unmarked markdown as marked output. + const { fetchImpl } = makeFetchFromSequence([ + { + matchUrl: /\/jobs$/, + matchMethod: "POST", + response: { + status: 200, + body: { + scrape_id: "scrape-id-test", + status: "done", + lane: "fast", + }, + }, + }, + { + matchUrl: /\/jobs\/scrape-id-test\/result$/, + matchMethod: "GET", + response: { + status: 200, + body: { + schema_version: 1, + markdown: "Page 1\n\n---\n\nPage 2", + pages_processed: 2, + }, + }, + }, + ]); + + const error = await scrapePDFWithFirePDFAsync( + makeMeta(), + "BASE64", + undefined, + undefined, + "auto", + { fetchImpl, sleepImpl: noopSleep }, + false, + false, + true, + ).catch(error => error); + + expect(error).toBeInstanceOf(FirePdfAsyncFailure); + expect(error.reason).toBe("http_5xx"); + }); + it("requests and returns typed blocks, tolerating the legacy pages alias", async () => { const blocks = [ { diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts index d697b38317..24c88fcd7e 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts @@ -82,6 +82,43 @@ describe("FirePDF page-markdown cache capabilities", () => { expect(cacheKeyShape("auto", 5, false, true).cacheable).toBe(false); }); + it("maps page-marker requests into a fully disjoint variant family", () => { + // pageMarkers rewrites the markdown itself, so no marker lookup may ever + // name a non-marker variant (and vice versa — see the plain-request test: + // its probe list is unchanged and marker-free). + expect(cacheKeyShape("auto", undefined, false, false, true)).toMatchObject({ + cacheable: true, + ownVariant: "markers-v1", + baseVariant: "markers-v1", + lookupVariants: [ + "markers-v1", + "page-markdown-markers-v1", + "ocr-markers-v1", + "ocr-page-markdown-markers-v1", + ], + }); + expect(cacheKeyShape("ocr", undefined, false, false, true)).toMatchObject({ + cacheable: true, + ownVariant: "ocr-markers-v1", + lookupVariants: ["ocr-markers-v1", "ocr-page-markdown-markers-v1"], + }); + expect(cacheKeyShape("auto", undefined, false, true, true)).toMatchObject({ + cacheable: true, + ownVariant: "blocks-markers-v1", + baseVariant: "markers-v1", + lookupVariants: [ + "blocks-markers-v1", + "page-markdown-blocks-markers-v1", + "ocr-blocks-markers-v1", + "ocr-page-markdown-blocks-markers-v1", + ], + }); + expect(cacheKeyShape("fast", undefined, false, false, true).cacheable).toBe( + false, + ); + expect(cacheKeyShape("auto", 5, false, false, true).cacheable).toBe(false); + }); + it("keeps the historical probe list for plain requests", () => { expect(cacheKeyShape("auto", undefined, false, false)).toMatchObject({ cacheable: true, @@ -504,4 +541,73 @@ describe("FirePDF page-markdown cache capabilities", () => { undefined, ); }); + + it("saves marker results only under marker variants (never the base key)", async () => { + const marked = "Page 1\n\n---\n\n\n\nPage 2"; + + await maybeSaveResult({ + meta: makeMeta(), + base64Content: "BASE64", + mode: "auto", + maxPages: undefined, + includePageMarkdown: false, + includeBlocks: false, + pageMarkers: true, + result: { + markdown: marked, + html: "

marked

", + pagesProcessed: 2, + }, + }); + + // Marker markdown must never back-fill the non-marker base key: a later + // plain request would silently receive marker-mutated markdown. + expect(saveCached).toHaveBeenCalledOnce(); + expect(saveCached).toHaveBeenCalledWith( + "BASE64", + expect.objectContaining({ markdown: marked }), + "firepdf", + "markers-v1", + ); + }); + + it("back-fills enriched marker results within the marker family only", async () => { + const marked = "Page 1\n\n---\n\n\n\nPage 2"; + const blocks = [ + { page: 1, width: 800, height: 1100, status: "ok", items: [] }, + ]; + + await maybeSaveResult({ + meta: makeMeta(), + base64Content: "BASE64", + mode: "auto", + maxPages: undefined, + includePageMarkdown: false, + includeBlocks: true, + pageMarkers: true, + result: { + markdown: marked, + html: "

marked

", + pagesProcessed: 2, + blocks, + }, + }); + + expect(getCached).toHaveBeenCalledWith("BASE64", "firepdf", "markers-v1"); + expect(saveCached).toHaveBeenCalledTimes(2); + expect(saveCached).toHaveBeenNthCalledWith( + 1, + "BASE64", + expect.objectContaining({ blocks }), + "firepdf", + "blocks-markers-v1", + ); + expect(saveCached).toHaveBeenNthCalledWith( + 2, + "BASE64", + expect.not.objectContaining({ blocks: expect.anything() }), + "firepdf", + "markers-v1", + ); + }); }); diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts index 5a12a42040..6e80a2c5f0 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/indexPageMarkdown.test.ts @@ -52,6 +52,15 @@ describe("PDF page-markdown URL index policy", () => { }, }), ).toBe(false); + expect( + shouldUseIndex({ + ...baseMeta, + options: { + ...baseMeta.options, + parsers: [{ type: "pdf", pageMarkers: true }], + }, + }), + ).toBe(false); } finally { ( config as { FIRECRAWL_INDEX_WRITE_ONLY?: boolean } @@ -116,4 +125,31 @@ describe("PDF page-markdown URL index policy", () => { expect(result).toBe(document); expect(result.metadata.indexId).toBeUndefined(); }); + + it("does not write marker-bearing markdown to the URL index", async () => { + const document = { + markdown: "Page 1\n\n---\n\n\n\nPage 2", + rawHtml: "

whole document

", + metadata: { + sourceURL: "https://example.com/file.pdf", + }, + } as any; + const meta = { + url: "https://example.com/file.pdf", + winnerEngine: "pdf", + options: { + storeInCache: true, + parsers: [{ type: "pdf", pageMarkers: true }], + }, + internalOptions: { + isParse: false, + zeroDataRetention: false, + }, + } as any; + + const result = await sendDocumentToIndex(meta, document); + + expect(result).toBe(document); + expect(result.metadata.indexId).toBeUndefined(); + }); }); diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts index 3174dc88ca..53e135bd6e 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts @@ -39,6 +39,7 @@ export async function scrapePDFWithFirePDFAsync( deps: FirePdfAsyncDeps = {}, includePageMarkdown = false, includeBlocks = false, + pageMarkers = false, ): Promise { const fetchImpl = deps.fetchImpl ?? undiciFetch; const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF; @@ -57,6 +58,7 @@ export async function scrapePDFWithFirePDFAsync( mode, includePageMarkdown, includeBlocks, + pageMarkers, ); } @@ -68,6 +70,7 @@ export async function scrapePDFWithFirePDFAsync( pagesProcessed, includePageMarkdown, includeBlocks, + pageMarkers, ); if (cached) return cached; @@ -93,6 +96,7 @@ export async function scrapePDFWithFirePDFAsync( mode, includePageMarkdown, includeBlocks, + pageMarkers, ); } @@ -127,6 +131,7 @@ export async function scrapePDFWithFirePDFAsync( mode, includePageMarkdown, includeBlocks, + pageMarkers, deadlineAt, teamConcurrency, fetchImpl, @@ -192,6 +197,16 @@ export async function scrapePDFWithFirePDFAsync( note: "FirePDF result omitted requested typed blocks", }); } + if (pageMarkers && fetched.page_markers !== true) { + // Markers are baked into the markdown, so the missing echo is the only + // signal the worker build ignored the option; accepting the result + // would cache unmarked markdown under a marker cache variant. Fail the + // async attempt — the caller retries synchronously, where the same + // echo contract applies. + failAsync(meta, "http_5xx", { + note: "FirePDF result did not acknowledge requested page markers", + }); + } const durationMs = now() - overallStartedAt; firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000); @@ -222,6 +237,7 @@ export async function scrapePDFWithFirePDFAsync( maxPages, includePageMarkdown, includeBlocks, + pageMarkers, result: processorResult, }); diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts index 647aaa3ae8..d62fc723e9 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts @@ -18,6 +18,19 @@ const OCR_BLOCKS_VARIANT = "ocr-blocks-v1"; const PAGE_MARKDOWN_BLOCKS_VARIANT = "page-markdown-blocks-v1"; const OCR_PAGE_MARKDOWN_BLOCKS_VARIANT = "ocr-page-markdown-blocks-v1"; +// `page_markers` rewrites the document markdown itself (inter-page +// `` separators), unlike pages/blocks which are extra +// payloads beside unchanged markdown. Marker and non-marker artifacts can +// therefore never serve each other. Follow the `mode: ocr` dedicated-variant +// precedent: map every variant name into a disjoint `…markers…` family. +// Within that family the ocr/pages/blocks capability lattice applies +// unchanged, because those artifacts differ only in sidecars again. +function withPageMarkers(variant: string | undefined): string { + if (variant === undefined) return "markers-v1"; + if (variant === "ocr") return "ocr-markers-v1"; + return variant.replace(/-v1$/, "-markers-v1"); +} + function isValidCachedDocument( value: unknown, ): value is Pick & { markdown: string } { @@ -67,6 +80,7 @@ export function cacheKeyShape( maxPages: number | undefined, includePageMarkdown: boolean, includeBlocks: boolean, + pageMarkers = false, ) { const cacheable = mode !== "fast" && !maxPages; const isOcr = mode === "ocr"; @@ -116,6 +130,14 @@ export function cacheKeyShape( : isOcr ? ["ocr", OCR_PAGE_MARKDOWN_VARIANT] : [undefined, PAGE_MARKDOWN_VARIANT, "ocr", OCR_PAGE_MARKDOWN_VARIANT]; + if (pageMarkers) { + return { + cacheable, + ownVariant: withPageMarkers(ownVariant), + baseVariant: withPageMarkers(baseVariant), + lookupVariants: lookupVariants.map(withPageMarkers), + }; + } return { cacheable, ownVariant, baseVariant, lookupVariants }; } @@ -127,6 +149,7 @@ export async function tryGetCached( pagesProcessed: number | undefined, includePageMarkdown: boolean, includeBlocks: boolean, + pageMarkers = false, ): Promise { if (meta.internalOptions.zeroDataRetention) return null; const { cacheable, lookupVariants } = cacheKeyShape( @@ -134,6 +157,7 @@ export async function tryGetCached( maxPages, includePageMarkdown, includeBlocks, + pageMarkers, ); if (!cacheable) return null; @@ -186,6 +210,7 @@ export async function maybeSaveResult(args: { maxPages: number | undefined; includePageMarkdown: boolean; includeBlocks: boolean; + pageMarkers?: boolean; result: PDFProcessorResult & { markdown: string }; }): Promise { const { @@ -195,6 +220,7 @@ export async function maybeSaveResult(args: { maxPages, includePageMarkdown, includeBlocks, + pageMarkers = false, result, } = args; if (meta.internalOptions.zeroDataRetention) return; @@ -203,6 +229,7 @@ export async function maybeSaveResult(args: { maxPages, includePageMarkdown, includeBlocks, + pageMarkers, ); if (!cacheable) return; diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts index 47f197e199..7e5bbf7341 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts @@ -125,6 +125,11 @@ export const resultResponseSchema = z.object({ pages_processed: z.number().optional(), failed_pages: z.array(z.number()).nullable().optional(), partial_pages: z.array(z.number()).nullable().optional(), + // Echo of an honored pageMarkers job option. Markers are baked into + // `markdown` and their absence is not detectable by content, so the echo + // is the only proof the fire-pdf worker build understood the option — + // older workers ignore unknown option keys and omit it. + page_markers: z.literal(true).optional(), }); export type PollResponse = z.infer; diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts index 968e0ad254..716cc47915 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts @@ -21,6 +21,7 @@ type SubmitArgs = { mode: PDFMode | undefined; includePageMarkdown: boolean; includeBlocks: boolean; + pageMarkers: boolean; deadlineAt: string; /** Team's sold concurrency from the ACUC (ENG-5049 account context). * Optional: entitlement lookup must never block or fail a scrape. */ @@ -62,6 +63,7 @@ export async function submitJob(args: SubmitArgs): Promise { mode, includePageMarkdown, includeBlocks, + pageMarkers, deadlineAt, teamConcurrency, fetchImpl, @@ -92,6 +94,12 @@ export async function submitJob(args: SubmitArgs): Promise { ...(mode !== undefined && { mode }), ...(includePageMarkdown && { include_page_markdown: true }), ...(includeBlocks && { include_blocks: true }), + // Intentionally camelCase, unlike its siblings: the fire-pdf async + // /jobs options schema named this key `pageMarkers` (fire-pdf + // api/src/http/schemas/jobs.ts) while the sync /ocr path uses + // `page_markers`. Sending snake_case here would be rejected as an + // unknown option. + ...(pageMarkers && { pageMarkers: true }), }, }; diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts index 3d9b11311c..de9d45779a 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts @@ -47,6 +47,7 @@ export async function scrapePDFWithFirePDF( mode?: PDFMode, includePageMarkdown = false, includeBlocks = false, + pageMarkers = false, ): Promise { const logger = meta.logger; @@ -62,6 +63,11 @@ export async function scrapePDFWithFirePDF( // running fire-pdf again. // - `fast` is bypassed entirely (hard cost ceiling — must fail on // scanned PDFs, not serve a cached OCR result). + // - `page_markers` rewrites the document markdown itself (inter-page + // `` separators), so marker requests read/write a + // fully disjoint `…markers…` variant family — a base-variant entry + // must never be served for a marker request and vice versa. See + // cacheKeyShape in fire-pdf/cache.ts. const cacheable = mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention; const cached = cacheable @@ -73,6 +79,7 @@ export async function scrapePDFWithFirePDF( pagesProcessed, includePageMarkdown, includeBlocks, + pageMarkers, ) : null; if (cached) return cached; @@ -124,6 +131,7 @@ export async function scrapePDFWithFirePDF( ...(mode !== undefined && { mode }), ...(includePageMarkdown && { include_page_markdown: true }), ...(includeBlocks && { include_blocks: true }), + ...(pageMarkers && { page_markers: true }), // Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats // these as optional — older fire-pdf builds will ignore unknown fields. team_id: meta.internalOptions.teamId, @@ -143,6 +151,12 @@ export async function scrapePDFWithFirePDF( pages_processed: z.number().optional(), pages: firePdfPagesSchema, blocks: firePdfBlocksSchema, + // Echo of an honored page_markers request. Markers are baked into + // `markdown` and their absence is not reliably detectable there (a + // single-page or fully-stitched document legitimately has none), so + // the echo is the only proof the fire-pdf build understood the + // option — older builds ignore unknown request fields and omit it. + page_markers: z.literal(true).optional(), }), mock: meta.mock, abort: meta.abort.asSignal(), @@ -157,6 +171,13 @@ export async function scrapePDFWithFirePDF( if (includeBlocks && resp.blocks === undefined) { throw new Error("FirePDF response did not include requested typed blocks"); } + if (pageMarkers && resp.page_markers !== true) { + // Without the echo, the markdown is ordinary unmarked output; caching + // it under a marker variant would silently poison the marker cache. + throw new Error( + "FirePDF response did not acknowledge requested page markers", + ); + } const pages = resp.pages_processed ?? pagesProcessed; logger.info("FirePDF completed", { @@ -185,6 +206,7 @@ export async function scrapePDFWithFirePDF( maxPages, includePageMarkdown, includeBlocks, + pageMarkers, result: processorResult, }); } diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts index e40c2ae194..bc67579ead 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts @@ -20,6 +20,7 @@ import { getPDFMode, getPDFPageMarkdown, getPDFBlocks, + getPDFPageMarkers, getFirePdfAsync, } from "../../../../controllers/v2/types"; import type { PDFMode } from "../../../../controllers/v2/types"; @@ -64,6 +65,7 @@ export async function scrapePDF(meta: Meta): Promise { const mode: PDFMode = getPDFMode(meta.options.parsers); const includePageMarkdown = getPDFPageMarkdown(meta.options.parsers); const includeBlocks = getPDFBlocks(meta.options.parsers); + const pageMarkers = getPDFPageMarkers(meta.options.parsers); if (includePageMarkdown && !config.FIRE_PDF_BASE_URL) { throw new Error( @@ -77,6 +79,12 @@ export async function scrapePDF(meta: Meta): Promise { ); } + if (pageMarkers && !config.FIRE_PDF_BASE_URL) { + throw new Error( + "Page markers are unavailable because FirePDF is not configured", + ); + } + if (!shouldParse) { if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) { const content = (await readFile(meta.pdfPrefetch.filePath)).toString( @@ -187,7 +195,10 @@ export async function scrapePDF(meta: Meta): Promise { let shadowPagesNeedingOcr: number[] | undefined; const forceFirePDF = - (!!meta.options.__forceFirePDF || includePageMarkdown || includeBlocks) && + (!!meta.options.__forceFirePDF || + includePageMarkdown || + includeBlocks || + pageMarkers) && !!config.FIRE_PDF_BASE_URL; const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE; const logger = meta.logger.child({ method: "scrapePDF/processPdf" }); @@ -471,17 +482,18 @@ export async function scrapePDF(meta: Meta): Promise { undefined, includePageMarkdown, includeBlocks, + pageMarkers, ); } catch (error) { if ( - (!includePageMarkdown && !includeBlocks) || + (!includePageMarkdown && !includeBlocks && !pageMarkers) || error instanceof RemoveFeatureError || error instanceof AbortManagerThrownError ) { throw error; } meta.logger.warn( - "FirePDF async page markdown/blocks failed -- retrying synchronously", + "FirePDF async page markdown/blocks/markers failed -- retrying synchronously", { method: "scrapePDF/firePDFFallback", error, @@ -502,6 +514,7 @@ export async function scrapePDF(meta: Meta): Promise { mode, includePageMarkdown, includeBlocks, + pageMarkers, ); } } else { @@ -513,6 +526,7 @@ export async function scrapePDF(meta: Meta): Promise { mode, includePageMarkdown, includeBlocks, + pageMarkers, ); } effectivePageCount = reconcilePageCountWithFirePdf( From 35a461f03da5e736e5e89ad5c0bfa2a146108685 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:58:47 -0700 Subject: [PATCH 2/2] feat(sdks): add parsers[].pageMarkers for PDF page attribution (#4379) Expose the v2 PDF parser pageMarkers option across the JS, Python, Go, Java, Ruby, PHP, Rust, and .NET SDKs so scrape/parse requests can join PDF pages in document.markdown with separators. Bump each package version so merge auto-publishes. Co-authored-by: Cursor Agent Co-authored-by: Abimael Martell --- .../Firecrawl.Tests/ModelsTests.cs | 19 +++++ apps/dot-net-sdk/Firecrawl/Firecrawl.csproj | 2 +- apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs | 2 +- .../dot-net-sdk/Firecrawl/Models/PdfParser.cs | 45 ++++++++++ apps/go-sdk/models.go | 17 ++-- apps/go-sdk/options_test.go | 5 +- apps/go-sdk/version.go | 2 +- apps/java-sdk/build.gradle.kts | 2 +- .../com/firecrawl/client/FirecrawlClient.java | 2 +- .../com/firecrawl/models/ParseOptions.java | 1 + .../java/com/firecrawl/models/PdfParser.java | 84 +++++++++++++++++++ .../com/firecrawl/models/ScrapeOptions.java | 2 +- .../com/firecrawl/FirecrawlClientTest.java | 19 +++++ apps/js-sdk/firecrawl/package.json | 2 +- .../src/__tests__/unit/v2/validation.test.ts | 5 +- apps/js-sdk/firecrawl/src/v2/types.ts | 8 ++ apps/php-sdk/CHANGELOG.md | 6 ++ apps/php-sdk/src/Models/PDFParser.php | 67 +++++++++++++++ apps/php-sdk/src/Models/ParseOptions.php | 11 ++- apps/php-sdk/src/Models/ScrapeOptions.php | 11 ++- apps/php-sdk/src/Version.php | 2 +- apps/php-sdk/tests/Unit/ModelsTest.php | 15 ++++ apps/python-sdk/firecrawl/__init__.py | 2 +- .../unit/v2/utils/test_validation.py | 12 ++- apps/python-sdk/firecrawl/v2/types.py | 5 ++ .../firecrawl/v2/utils/validation.py | 4 + apps/ruby-sdk/lib/firecrawl.rb | 1 + .../lib/firecrawl/models/parse_options.rb | 2 +- .../lib/firecrawl/models/pdf_parser.rb | 33 ++++++++ .../lib/firecrawl/models/scrape_options.rb | 2 +- apps/ruby-sdk/lib/firecrawl/version.rb | 2 +- apps/ruby-sdk/test/firecrawl/client_test.rb | 32 +++++++ apps/rust-sdk/CHANGELOG.md | 7 ++ apps/rust-sdk/Cargo.lock | 2 +- apps/rust-sdk/Cargo.toml | 2 +- apps/rust-sdk/src/scrape.rs | 7 +- 36 files changed, 407 insertions(+), 35 deletions(-) create mode 100644 apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs create mode 100644 apps/java-sdk/src/main/java/com/firecrawl/models/PdfParser.java create mode 100644 apps/php-sdk/src/Models/PDFParser.php create mode 100644 apps/ruby-sdk/lib/firecrawl/models/pdf_parser.rb diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs index ec8e9c71ec..612fbe99c9 100644 --- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs +++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs @@ -426,6 +426,25 @@ public void CrawlJob_DeserializesCorrectly() Assert.Equal(2, job.Data.Count); } + [Fact] + public void PdfParser_SerializesPageMarkers() + { + var parser = new PdfParser + { + Mode = "auto", + Pages = true, + Blocks = true, + PageMarkers = true + }; + + var json = JsonSerializer.Serialize(parser, JsonOptions); + Assert.Contains("\"type\":\"pdf\"", json); + Assert.Contains("\"mode\":\"auto\"", json); + Assert.Contains("\"pages\":true", json); + Assert.Contains("\"blocks\":true", json); + Assert.Contains("\"pageMarkers\":true", json); + } + [Fact] public void JsonFormat_HasCorrectType() { diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj index a1da003731..7bc41f43a8 100644 --- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj +++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj @@ -8,7 +8,7 @@ firecrawl-sdk - 1.12.0 + 1.13.0 Firecrawl Firecrawl .NET SDK for the Firecrawl API - web scraping, crawling, and data extraction diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs index e3bcb1a65e..169db89f80 100644 --- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs +++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs @@ -714,7 +714,7 @@ private async Task PaginateMonitorCheckAsync( // INTERNAL UTILITIES // ================================================================ - private const string SdkOrigin = "dotnet-sdk@1.12.0"; + private const string SdkOrigin = "dotnet-sdk@1.13.0"; private static Dictionary BuildBody(object? options) { diff --git a/apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs b/apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs new file mode 100644 index 0000000000..f8f393c78e --- /dev/null +++ b/apps/dot-net-sdk/Firecrawl/Models/PdfParser.cs @@ -0,0 +1,45 @@ +using System.Text.Json.Serialization; + +namespace Firecrawl.Models; + +/// +/// PDF parser configuration for use in / +/// . +/// +public class PdfParser +{ + [JsonPropertyName("type")] + public string Type { get; } = "pdf"; + + [JsonPropertyName("mode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Mode { get; set; } + + [JsonPropertyName("maxPages")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxPages { get; set; } + + /// + /// Include physical per-page markdown. Populates document.pages. + /// + [JsonPropertyName("pages")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Pages { get; set; } + + /// + /// Include per-page typed layout blocks. Populates document.blocks. + /// + [JsonPropertyName("blocks")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Blocks { get; set; } + + /// + /// Join PDF pages in document.markdown with + /// \n\n---\n\n<!-- page N -->\n\n. Markers appear between + /// pages only; numbering may skip pages merged by cross-page stitching — + /// use when every physical page is needed. + /// + [JsonPropertyName("pageMarkers")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? PageMarkers { get; set; } +} diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go index be40771720..f65ff09189 100644 --- a/apps/go-sdk/models.go +++ b/apps/go-sdk/models.go @@ -37,6 +37,11 @@ type PDFParser struct { MaxPages *int `json:"maxPages,omitempty"` Pages *bool `json:"pages,omitempty"` Blocks *bool `json:"blocks,omitempty"` + // PageMarkers joins PDF pages in document.markdown with + // `\n\n---\n\n\n\n`. Markers appear between pages only; + // numbering may skip pages merged by cross-page stitching — use Pages + // when every physical page is needed. No new response field. + PageMarkers *bool `json:"pageMarkers,omitempty"` } // PdfPage is physical markdown for a single PDF page. @@ -76,12 +81,12 @@ type PdfPageBlocks struct { // ProductProfile represents structured product data extracted from a page // via the `product` scrape format. type ProductProfile struct { - Title string `json:"title"` - Brand string `json:"brand,omitempty"` - Category string `json:"category,omitempty"` - URL string `json:"url"` - Description string `json:"description,omitempty"` - Variants []ProductVariant `json:"variants,omitempty"` + Title string `json:"title"` + Brand string `json:"brand,omitempty"` + Category string `json:"category,omitempty"` + URL string `json:"url"` + Description string `json:"description,omitempty"` + Variants []ProductVariant `json:"variants,omitempty"` } // ProductImage is a single product image. diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go index 4f8db40137..40c80df091 100644 --- a/apps/go-sdk/options_test.go +++ b/apps/go-sdk/options_test.go @@ -67,16 +67,17 @@ func TestScrapeOptionsPreservesStringFormats(t *testing.T) { func TestScrapeOptionsSerializesPDFParserPagesAndBlocks(t *testing.T) { pages := true blocks := true + pageMarkers := true payload, err := json.Marshal(ScrapeOptions{ Parsers: []interface{}{ - PDFParser{Type: "pdf", Mode: "auto", Pages: &pages, Blocks: &blocks}, + PDFParser{Type: "pdf", Mode: "auto", Pages: &pages, Blocks: &blocks, PageMarkers: &pageMarkers}, }, }) if err != nil { t.Fatalf("Marshal ScrapeOptions: %v", err) } - want := `"parsers":[{"type":"pdf","mode":"auto","pages":true,"blocks":true}]` + want := `"parsers":[{"type":"pdf","mode":"auto","pages":true,"blocks":true,"pageMarkers":true}]` if !strings.Contains(string(payload), want) { t.Fatalf("serialized parsers = %s, want to contain %s", payload, want) } diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go index c27aa29280..186497366a 100644 --- a/apps/go-sdk/version.go +++ b/apps/go-sdk/version.go @@ -9,4 +9,4 @@ package firecrawl // Bump this when preparing a new release. The publish-go-sdk GitHub workflow // reads this value and creates the corresponding monorepo-prefixed tag on // merge to main. -const Version = "1.11.0" +const Version = "1.12.0" diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts index e9c98c03a6..3d268db2a8 100644 --- a/apps/java-sdk/build.gradle.kts +++ b/apps/java-sdk/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } group = "com.firecrawl" -version = "1.14.0" +version = "1.15.0" java { sourceCompatibility = JavaVersion.VERSION_11 diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java index f0c9f8ada7..3b725f3b36 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java @@ -37,7 +37,7 @@ public class FirecrawlClient { private static final String DEFAULT_API_URL = "https://api.firecrawl.dev"; - private static final String SDK_ORIGIN = "java-sdk@1.14.0"; + private static final String SDK_ORIGIN = "java-sdk@1.15.0"; private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutes private static final int DEFAULT_MAX_RETRIES = 3; private static final double DEFAULT_BACKOFF_FACTOR = 0.5; diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java index 01b133df1c..ff88c6f5fa 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java @@ -127,6 +127,7 @@ private Builder() {} public Builder excludeTags(List excludeTags) { this.excludeTags = excludeTags; return this; } public Builder onlyMainContent(Boolean onlyMainContent) { this.onlyMainContent = onlyMainContent; return this; } public Builder timeout(Integer timeout) { this.timeout = timeout; return this; } + /** Parsers to use (e.g., "pdf" or PdfParser with maxPages, pages, blocks, pageMarkers). */ public Builder parsers(List parsers) { this.parsers = parsers; return this; } public Builder skipTlsVerification(Boolean skipTlsVerification) { this.skipTlsVerification = skipTlsVerification; return this; } public Builder removeBase64Images(Boolean removeBase64Images) { this.removeBase64Images = removeBase64Images; return this; } diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/PdfParser.java b/apps/java-sdk/src/main/java/com/firecrawl/models/PdfParser.java new file mode 100644 index 0000000000..a32db3f82d --- /dev/null +++ b/apps/java-sdk/src/main/java/com/firecrawl/models/PdfParser.java @@ -0,0 +1,84 @@ +package com.firecrawl.models; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * PDF parser configuration for use in {@code ScrapeOptions.parsers} / + * {@code ParseOptions.parsers}. + * + *

Usage: + *

{@code
+ * PdfParser pdf = PdfParser.builder()
+ *     .mode("auto")
+ *     .pages(true)
+ *     .blocks(true)
+ *     .pageMarkers(true)
+ *     .build();
+ * }
+ */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PdfParser { + + private final String type = "pdf"; + private String mode; + @JsonProperty("maxPages") + private Integer maxPages; + private Boolean pages; + private Boolean blocks; + @JsonProperty("pageMarkers") + private Boolean pageMarkers; + + private PdfParser() {} + + public String getType() { return type; } + public String getMode() { return mode; } + @JsonProperty("maxPages") + public Integer getMaxPages() { return maxPages; } + public Boolean getPages() { return pages; } + public Boolean getBlocks() { return blocks; } + @JsonProperty("pageMarkers") + public Boolean getPageMarkers() { return pageMarkers; } + + public static Builder builder() { return new Builder(); } + + public static final class Builder { + private String mode; + private Integer maxPages; + private Boolean pages; + private Boolean blocks; + private Boolean pageMarkers; + + private Builder() {} + + /** PDF processing mode: {@code "fast"}, {@code "auto"}, or {@code "ocr"}. */ + public Builder mode(String mode) { this.mode = mode; return this; } + + /** Maximum number of PDF pages to process. */ + public Builder maxPages(Integer maxPages) { this.maxPages = maxPages; return this; } + + /** Include physical per-page markdown. Populates {@code document.pages}. */ + public Builder pages(Boolean pages) { this.pages = pages; return this; } + + /** Include per-page typed layout blocks. Populates {@code document.blocks}. */ + public Builder blocks(Boolean blocks) { this.blocks = blocks; return this; } + + /** + * Join PDF pages in {@code document.markdown} with + * {@code \n\n---\n\n\n\n}. Markers appear between pages + * only; numbering may skip pages merged by cross-page stitching — use + * {@code pages(true)} when every physical page is needed. + */ + public Builder pageMarkers(Boolean pageMarkers) { this.pageMarkers = pageMarkers; return this; } + + public PdfParser build() { + PdfParser p = new PdfParser(); + p.mode = this.mode; + p.maxPages = this.maxPages; + p.pages = this.pages; + p.blocks = this.blocks; + p.pageMarkers = this.pageMarkers; + return p; + } + } +} diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java index 202c27214c..8381b554a8 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java @@ -145,7 +145,7 @@ private Builder() {} /** Scrape as a mobile device. */ public Builder mobile(Boolean mobile) { this.mobile = mobile; return this; } - /** Parsers to use (e.g., "pdf" or {"type": "pdf", "maxPages": 10, "pages": true, "blocks": true}). */ + /** Parsers to use (e.g., "pdf" or PdfParser with maxPages, pages, blocks, pageMarkers). */ public Builder parsers(List parsers) { this.parsers = parsers; return this; } /** Actions to execute before/during scraping. */ diff --git a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java index c2cff9d5c9..1cefffcdd4 100644 --- a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java +++ b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java @@ -261,6 +261,25 @@ void testParseOptionsRejectsMenuFormat() { ); } + @Test + void testPdfParserSerializesPageMarkers() throws Exception { + PdfParser parser = PdfParser.builder() + .mode("auto") + .pages(true) + .blocks(true) + .pageMarkers(true) + .build(); + + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + String json = mapper.writeValueAsString(parser); + + assertTrue(json.contains("\"type\":\"pdf\"")); + assertTrue(json.contains("\"mode\":\"auto\"")); + assertTrue(json.contains("\"pages\":true")); + assertTrue(json.contains("\"blocks\":true")); + assertTrue(json.contains("\"pageMarkers\":true")); + } + @Test void testDocumentDeserializesPages() throws Exception { String json = "{\"markdown\":\"# Annual Report 2025\",\"pages\":[" diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index a301ffd2f3..276ed946b2 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.34.2", + "version": "4.35.0", "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts index 162558e4fe..542078e87f 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts @@ -89,9 +89,9 @@ describe("v2 utils: validation", () => { expect(options.parsers).toEqual(before); }); - test("ensureValidScrapeOptions: leaves PDF parser pages and blocks options untouched", () => { + test("ensureValidScrapeOptions: leaves PDF parser pages, blocks, and pageMarkers options untouched", () => { const options = { - parsers: [{ type: "pdf", mode: "auto", blocks: true, pages: true }], + parsers: [{ type: "pdf", mode: "auto", blocks: true, pages: true, pageMarkers: true }], } as any; expect(() => ensureValidScrapeOptions(options)).not.toThrow(); expect(options.parsers[0]).toEqual({ @@ -99,6 +99,7 @@ describe("v2 utils: validation", () => { mode: "auto", blocks: true, pages: true, + pageMarkers: true, }); }); diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index 1c6b76e2f2..a3a9d4eea6 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -199,6 +199,14 @@ export type PDFParser = { pageMarkdown?: boolean; /** Include per-page typed layout blocks (bounding boxes, block types, reading order). */ blocks?: boolean; + /** + * Join PDF pages in `document.markdown` with `\n\n---\n\n\n\n`, + * where N is the 1-based physical page of the content that follows. Markers + * appear between pages only (no leading marker for page 1), and numbering may + * skip pages merged by cross-page stitching — use `pages: true` when every + * physical page is needed. No new response field. + */ + pageMarkers?: boolean; }; export interface PdfPage { diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.md index 96660ff6be..8ff0f27789 100644 --- a/apps/php-sdk/CHANGELOG.md +++ b/apps/php-sdk/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to the Firecrawl PHP SDK will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.13.0] - 2026-08-21 + +### Added +- PDF parser `pageMarkers` option via `PDFParser` to join pages in + document markdown with `` separators. + ## [1.12.0] - 2026-08-19 ### Added diff --git a/apps/php-sdk/src/Models/PDFParser.php b/apps/php-sdk/src/Models/PDFParser.php new file mode 100644 index 0000000000..4e2dada64d --- /dev/null +++ b/apps/php-sdk/src/Models/PDFParser.php @@ -0,0 +1,67 @@ + */ + public function toArray(): array + { + return array_filter([ + 'type' => 'pdf', + 'mode' => $this->mode, + 'maxPages' => $this->maxPages, + 'pages' => $this->pages, + 'blocks' => $this->blocks, + 'pageMarkers' => $this->pageMarkers, + ], fn (mixed $v): bool => $v !== null); + } + + public function getMode(): ?string + { + return $this->mode; + } + + public function getMaxPages(): ?int + { + return $this->maxPages; + } + + public function getPages(): ?bool + { + return $this->pages; + } + + public function getBlocks(): ?bool + { + return $this->blocks; + } + + public function getPageMarkers(): ?bool + { + return $this->pageMarkers; + } +} diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.php index 06c14db894..dc8a7396ab 100644 --- a/apps/php-sdk/src/Models/ParseOptions.php +++ b/apps/php-sdk/src/Models/ParseOptions.php @@ -31,7 +31,7 @@ final class ParseOptions * @param array|null $headers * @param list|null $includeTags * @param list|null $excludeTags - * @param list|null $parsers + * @param list>|null $parsers * @param AuditMetadata|null $auditMetadata */ private function __construct( @@ -56,7 +56,7 @@ private function __construct( * @param array|null $headers * @param list|null $includeTags * @param list|null $excludeTags - * @param list|null $parsers + * @param list>|null $parsers * @param AuditMetadata|null $auditMetadata */ public static function with( @@ -134,7 +134,10 @@ public function toArray(): array 'excludeTags' => $this->excludeTags, 'onlyMainContent' => $this->onlyMainContent, 'timeout' => $this->timeout, - 'parsers' => $this->parsers, + 'parsers' => $this->parsers === null ? null : array_map( + fn (mixed $parser): mixed => $parser instanceof PDFParser ? $parser->toArray() : $parser, + $this->parsers, + ), 'skipTlsVerification' => $this->skipTlsVerification, 'removeBase64Images' => $this->removeBase64Images, 'blockAds' => $this->blockAds, @@ -210,7 +213,7 @@ public function getTimeout(): ?int return $this->timeout; } - /** @return list|null */ + /** @return list>|null */ public function getParsers(): ?array { return $this->parsers; diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.php index 5cc3ad42b4..5b35b4372b 100644 --- a/apps/php-sdk/src/Models/ScrapeOptions.php +++ b/apps/php-sdk/src/Models/ScrapeOptions.php @@ -11,7 +11,7 @@ final class ScrapeOptions * @param array|null $headers * @param list|null $includeTags * @param list|null $excludeTags - * @param list|null $parsers + * @param list>|null $parsers * @param list>|null $actions * @param AuditMetadata|null $auditMetadata */ @@ -48,7 +48,7 @@ private function __construct( * @param array|null $headers * @param list|null $includeTags * @param list|null $excludeTags - * @param list|null $parsers + * @param list>|null $parsers * @param list>|null $actions * @param array|null $profile * @param AuditMetadata|null $auditMetadata @@ -115,7 +115,10 @@ public function toArray(): array 'timeout' => $this->timeout, 'waitFor' => $this->waitFor, 'mobile' => $this->mobile, - 'parsers' => $this->parsers, + 'parsers' => $this->parsers === null ? null : array_map( + fn (mixed $parser): mixed => $parser instanceof PDFParser ? $parser->toArray() : $parser, + $this->parsers, + ), 'actions' => $this->actions, 'location' => $this->location?->toArray(), 'skipTlsVerification' => $this->skipTlsVerification, @@ -191,7 +194,7 @@ public function getRedactPII(): ?bool return $this->redactPII; } - /** @return list|null */ + /** @return list>|null */ public function getParsers(): ?array { return $this->parsers; diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php index 15e2cf9965..82890da062 100644 --- a/apps/php-sdk/src/Version.php +++ b/apps/php-sdk/src/Version.php @@ -6,5 +6,5 @@ final class Version { - public const SDK_VERSION = '1.12.0'; + public const SDK_VERSION = '1.13.0'; } diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php index 8a11fd0c60..30b0fa8786 100644 --- a/apps/php-sdk/tests/Unit/ModelsTest.php +++ b/apps/php-sdk/tests/Unit/ModelsTest.php @@ -14,6 +14,7 @@ use Firecrawl\Models\AuditMetadata; use Firecrawl\Models\MapOptions; use Firecrawl\Models\ParseOptions; +use Firecrawl\Models\PDFParser; use Firecrawl\Models\QueryFormat; use Firecrawl\Models\QuestionFormat; use Firecrawl\Models\ScrapeOptions; @@ -361,6 +362,20 @@ ]); }); +it('serializes PDF parser pageMarkers in ScrapeOptions', function (): void { + $options = ScrapeOptions::with( + parsers: [PDFParser::with(mode: 'auto', pages: true, blocks: true, pageMarkers: true)], + ); + + expect($options->toArray()['parsers'][0])->toMatchArray([ + 'type' => 'pdf', + 'mode' => 'auto', + 'pages' => true, + 'blocks' => true, + 'pageMarkers' => true, + ]); +}); + it('serializes lockdown in ScrapeOptions', function (): void { $options = ScrapeOptions::with( lockdown: true, diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index c4babac4c8..ef319169ff 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -18,7 +18,7 @@ V1ChangeTrackingOptions, ) -__version__ = "4.37.1" +__version__ = "4.38.0" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py index 1f9c6f9840..24d1e0bc15 100644 --- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py +++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py @@ -390,8 +390,8 @@ def test_prepare_parsers_max_pages_model(self): assert result["parsers"][0]["maxPages"] == 5 def test_prepare_parsers_blocks_and_pages(self): - """PDF parser blocks and pages are forwarded to the API.""" - parser = PDFParser(mode="auto", pages=True, blocks=True) + """PDF parser blocks, pages, and page_markers are forwarded to the API.""" + parser = PDFParser(mode="auto", pages=True, blocks=True, page_markers=True) options = ScrapeOptions(parsers=[parser]) result = prepare_scrape_options(options) @@ -401,8 +401,16 @@ def test_prepare_parsers_blocks_and_pages(self): "mode": "auto", "pages": True, "blocks": True, + "pageMarkers": True, } + markers_options = ScrapeOptions( + parsers=[{"type": "pdf", "page_markers": True}] + ) + markers_result = prepare_scrape_options(markers_options) + assert markers_result["parsers"][0]["pageMarkers"] is True + assert "page_markers" not in markers_result["parsers"][0] + dict_options = ScrapeOptions( parsers=[{"type": "pdf", "page_markdown": True, "blocks": True}] ) diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index 4eeab4e9f8..6690496aba 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -1671,6 +1671,11 @@ class PDFParser(BaseModel): max_pages: Optional[int] = None pages: Optional[bool] = None blocks: Optional[bool] = None + # Join PDF pages in document markdown with `\n\n---\n\n\n\n` + # (N = 1-based physical page of the content that follows). Markers appear + # between pages only, and numbering may skip pages merged by cross-page + # stitching — use `pages=True` when every physical page is needed. + page_markers: Optional[bool] = None @model_validator(mode="before") @classmethod diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py index f88dfa06ed..b8c4bc05ae 100644 --- a/apps/python-sdk/firecrawl/v2/utils/validation.py +++ b/apps/python-sdk/firecrawl/v2/utils/validation.py @@ -795,12 +795,16 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st parser_data.setdefault("pages", parser_data.pop("page_markdown")) if "pageMarkdown" in parser_data: parser_data.setdefault("pages", parser_data.pop("pageMarkdown")) + if "page_markers" in parser_data: + parser_data["pageMarkers"] = parser_data.pop("page_markers") converted_parsers.append(parser_data) else: parser_data = parser.model_dump(exclude_none=True) # Convert snake_case to camelCase for API if "max_pages" in parser_data: parser_data["maxPages"] = parser_data.pop("max_pages") + if "page_markers" in parser_data: + parser_data["pageMarkers"] = parser_data.pop("page_markers") converted_parsers.append(parser_data) scrape_data["parsers"] = converted_parsers elif key == "location": diff --git a/apps/ruby-sdk/lib/firecrawl.rb b/apps/ruby-sdk/lib/firecrawl.rb index 3ba8cd0dd1..b6398d36c4 100644 --- a/apps/ruby-sdk/lib/firecrawl.rb +++ b/apps/ruby-sdk/lib/firecrawl.rb @@ -4,6 +4,7 @@ require_relative "firecrawl/errors" require_relative "firecrawl/http_client" require_relative "firecrawl/models/query_format" +require_relative "firecrawl/models/pdf_parser" require_relative "firecrawl/models/product_profile" require_relative "firecrawl/models/menu_profile" require_relative "firecrawl/models/document" diff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb index cb7310e95a..8892a5210d 100644 --- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb +++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb @@ -35,7 +35,7 @@ def to_h "excludeTags" => exclude_tags, "onlyMainContent" => only_main_content, "timeout" => timeout, - "parsers" => parsers, + "parsers" => parsers&.map { |parser| parser.respond_to?(:to_h) ? parser.to_h : parser }, "skipTlsVerification" => skip_tls_verification, "removeBase64Images" => remove_base64_images, "blockAds" => block_ads, diff --git a/apps/ruby-sdk/lib/firecrawl/models/pdf_parser.rb b/apps/ruby-sdk/lib/firecrawl/models/pdf_parser.rb new file mode 100644 index 0000000000..7fec9afd33 --- /dev/null +++ b/apps/ruby-sdk/lib/firecrawl/models/pdf_parser.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Firecrawl + module Models + # PDF parser configuration for use in ScrapeOptions / ParseOptions parsers. + class PDFParser + attr_reader :mode, :max_pages, :pages, :blocks, :page_markers + + def initialize(mode: nil, max_pages: nil, pages: nil, blocks: nil, page_markers: nil) + @mode = mode + @max_pages = max_pages + @pages = pages + @blocks = blocks + @page_markers = page_markers + end + + def to_h + { + "type" => "pdf", + "mode" => mode, + "maxPages" => max_pages, + "pages" => pages, + "blocks" => blocks, + "pageMarkers" => page_markers, + }.compact + end + + def type + "pdf" + end + end + end +end diff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb index 20c0505556..10049e813e 100644 --- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb +++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb @@ -31,7 +31,7 @@ def to_h "timeout" => timeout, "waitFor" => wait_for, "mobile" => mobile, - "parsers" => parsers, + "parsers" => parsers&.map { |parser| parser.respond_to?(:to_h) ? parser.to_h : parser }, "actions" => actions, "location" => location.is_a?(Hash) ? location : location&.to_h, "skipTlsVerification" => skip_tls_verification, diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb index ad35645727..b14207c3ee 100644 --- a/apps/ruby-sdk/lib/firecrawl/version.rb +++ b/apps/ruby-sdk/lib/firecrawl/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Firecrawl - VERSION = "1.13.0" + VERSION = "1.14.0" end diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb index 644cf0c66d..5148e8f98d 100644 --- a/apps/ruby-sdk/test/firecrawl/client_test.rb +++ b/apps/ruby-sdk/test/firecrawl/client_test.rb @@ -135,6 +135,38 @@ def test_scrape_hydrates_pdf_blocks assert_equal "title", doc.blocks[0]["items"][0]["type"] end + def test_scrape_serializes_pdf_parser_page_markers + stub_request(:post, "#{BASE_URL}/v2/scrape") + .with { |req| + body = JSON.parse(req.body) + body["parsers"] == [{ + "type" => "pdf", + "mode" => "auto", + "pages" => true, + "blocks" => true, + "pageMarkers" => true + }] + } + .to_return( + status: 200, + body: JSON.generate(data: { markdown: "# Cover" }), + headers: { "Content-Type" => "application/json" } + ) + + options = Firecrawl::Models::ScrapeOptions.new( + parsers: [ + Firecrawl::Models::PDFParser.new( + mode: "auto", + pages: true, + blocks: true, + page_markers: true + ) + ] + ) + doc = @client.scrape("https://example.com/report.pdf", options) + assert_equal "# Cover", doc.markdown + end + def test_scrape_with_options stub_request(:post, "#{BASE_URL}/v2/scrape") .with { |req| body = JSON.parse(req.body); body["formats"] == ["markdown", "html"] && body["onlyMainContent"] == true } diff --git a/apps/rust-sdk/CHANGELOG.md b/apps/rust-sdk/CHANGELOG.md index 099a60669f..4a1853b4f0 100644 --- a/apps/rust-sdk/CHANGELOG.md +++ b/apps/rust-sdk/CHANGELOG.md @@ -1,5 +1,12 @@ ## CHANGELOG +## [2.16.0] - 2026-08-21 + +### Added + +- Added `ParserConfig::Pdf.page_markers` to join PDF pages in + `document.markdown` with `\n\n---\n\n\n\n`. + ## [2.15.0] - 2026-08-21 ### Added diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index 35d3e537c9..1aa48bbb18 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "firecrawl" -version = "2.15.0" +version = "2.16.0" dependencies = [ "mockito", "reqwest", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index a86a818856..a5b6647212 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecrawl" -version = "2.15.0" +version = "2.16.0" edition = "2021" license = "MIT" homepage = "https://www.firecrawl.dev/" diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs index 29374e4147..9372dcbd93 100644 --- a/apps/rust-sdk/src/scrape.rs +++ b/apps/rust-sdk/src/scrape.rs @@ -123,6 +123,9 @@ pub enum ParserConfig { pages: Option, #[serde(skip_serializing_if = "Option::is_none")] blocks: Option, + /// Join PDF pages in document markdown with `\n\n---\n\n\n\n`. + #[serde(rename = "pageMarkers", skip_serializing_if = "Option::is_none")] + page_markers: Option, }, } @@ -533,6 +536,7 @@ mod tests { max_pages: None, pages: Some(true), blocks: Some(true), + page_markers: Some(true), }]), ..Default::default() }; @@ -544,7 +548,8 @@ mod tests { "type": "pdf", "mode": "auto", "pages": true, - "blocks": true + "blocks": true, + "pageMarkers": true }) ); }