From 10d9c767453967ea61fc513a1feb6f786f536180 Mon Sep 17 00:00:00 2001 From: "Pieter J. Kersten" Date: Mon, 24 Aug 2026 09:57:17 +0200 Subject: [PATCH] feat: scope page ranges to match and fields --- docs/cookbook.md | 39 +++++++++- src/invoice2data/api.py | 13 +++- src/invoice2data/extract/invoice_template.py | 19 +++-- src/invoice2data/extract/loader.py | 36 +++++++++ tests/test_page_ranges.py | 79 ++++++++++++++++++++ 5 files changed, 176 insertions(+), 10 deletions(-) diff --git a/docs/cookbook.md b/docs/cookbook.md index b8d5c95c..34e9ae41 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -205,10 +205,45 @@ Backends that support area: `pdftotext` (native, uses `-x/-y/-W/-H`) and `input_module: pdftotext` at the top level (bundled area templates already do this). +## "An envelope identifies the supplier; the invoice is on later pages" + +Scope every text selector independently. This is the preferred form for a +multi-page document whose envelope, payment reference, invoice header and +line items live on different pages: + +```yaml +issuer: Example supplier +input_module: pdftotext +match: + pages: 1 + keywords: + - Example supplier +fields: + invoice_number: + parser: regex + pages: "2-3" + regex: 'Invoice number\\s+(\\S+)' + payment_reference: + parser: regex + pages: 1 + regex: 'Reference\\s+(\\S+)' + lines: + parser: lines + pages: "2-3" + line: '...' +``` + +`match.pages` only gates template selection. A field's `pages` limits only +that field, and combines with its `area` when both are set. This applies to +modern `fields:` parsers, including `fields.lines`. Camelot is already +page-aware through its own table setting. It is intentionally not attached to +static values or other settings that do not read document text. + ## "Only certain PDF pages are the invoice" -Use a top-level inclusive `pages:` range. Unlike a field `area:`, this limits -template keywords, document fields, field areas and line extraction together: +For an existing homogeneous template, the top-level inclusive `pages:` range +remains available as a compatibility shortcut. It limits template keywords, +document fields, field areas and line extraction together: ```yaml issuer: Example supplier diff --git a/src/invoice2data/api.py b/src/invoice2data/api.py index defc8960..69eb6c1e 100644 --- a/src/invoice2data/api.py +++ b/src/invoice2data/api.py @@ -285,10 +285,14 @@ def _match_template( def _match_template_for_reader( extracted_str: str, templates: list[InvoiceTemplate], invoicefile: str, reader: Any ) -> tuple[InvoiceTemplate | None, str]: - """Match templates, applying a template's optional page scope first.""" + """Match templates, applying the match selector's optional page scope.""" for template in templates: scoped_text = extracted_str - pages = template.get("pages") + match = template.get("match") + # ``pages`` at the template root is the pre-1.1 compatibility form. + # New templates make the matching scope explicit in ``match.pages``; + # their fields select their own text independently. + pages = match.get("pages") if isinstance(match, dict) else template.get("pages") if pages is not None: try: scoped_text = extract_text(reader, invoicefile, pages=pages) @@ -302,7 +306,10 @@ def _match_template_for_reader( ) continue if template.matches_input(scoped_text): - return template, scoped_text + # A selector-based template may match an envelope while extracting + # its invoice fields from later pages. Do not leak its match scope + # into those fields; each field owns its own optional ``pages``. + return template, extracted_str if isinstance(match, dict) else scoped_text return None, extracted_str diff --git a/src/invoice2data/extract/invoice_template.py b/src/invoice2data/extract/invoice_template.py index 9622f335..9ead9c1a 100644 --- a/src/invoice2data/extract/invoice_template.py +++ b/src/invoice2data/extract/invoice_template.py @@ -269,17 +269,24 @@ def extract( for k, v in self["fields"].items(): if isinstance(v, dict): + parser_settings = dict(v) + # A modern field is a text selector in its own right. Keep + # ``pages`` out of parser settings: it controls input text, + # rather than the regex/lines parser's behaviour. + pages = parser_settings.pop("pages", self.get("pages")) optimized_str_for_parser = _handle_area( self, - v, + parser_settings, input_module, invoice_file, optimized_str, - self.get("pages"), + pages, ) - if "parser" in v: - _handle_parser(self, k, v, optimized_str_for_parser, output) + if "parser" in parser_settings: + _handle_parser( + self, k, parser_settings, optimized_str_for_parser, output + ) elif k.startswith("static_"): logger.debug("field=%s | static value=%s", k, v) @@ -343,7 +350,7 @@ def _handle_area( optimized_str: str, pages: Any = None, ) -> str: - """Handle area-specific extraction.""" + """Select a field's optional page range and/or physical area.""" if "area" in v and supports_area(input_module): logger.debug(f"Area was specified with parameters {v['area']}") optimized_str_area: str = extract_text( @@ -355,6 +362,8 @@ def _handle_area( ) logger.debug("END pdftotext area result =============================") return optimized_str_area + if pages is not None: + return extract_text(input_module, invoice_file, pages=pages) return optimized_str diff --git a/src/invoice2data/extract/loader.py b/src/invoice2data/extract/loader.py index c686bad1..2528c7b3 100644 --- a/src/invoice2data/extract/loader.py +++ b/src/invoice2data/extract/loader.py @@ -217,6 +217,39 @@ def read_templates(folder: str | None = None) -> list[InvoiceTemplate]: return list(_read_templates_cached(folder, _folder_signature(folder))) +def _prepare_match_selector(tpl: dict[str, Any]) -> bool: + """Normalize an explicit ``match`` selector into legacy internal keys.""" + match = tpl.get("match") + if match is None: + return True + if not isinstance(match, dict): + logger.warning( + "Failed to load template %s. 'match' must be a mapping.", + tpl.get("template_name", ""), + ) + return False + if any(key in tpl for key in ("keywords", "exclude_keywords", "pages")): + logger.warning( + "Failed to load template %s. Put keywords, exclude_keywords and " + "pages inside 'match', not alongside it.", + tpl.get("template_name", ""), + ) + return False + if "keywords" not in match: + logger.warning( + "Failed to load template %s. 'match' is missing mandatory 'keywords'.", + tpl.get("template_name", ""), + ) + return False + # Keep the established internal representation. The nested selector is + # retained for its page scope while InvoiceTemplate continues to expose + # keywords at the top level to existing callers and plugins. + tpl["keywords"] = match["keywords"] + if "exclude_keywords" in match: + tpl["exclude_keywords"] = match["exclude_keywords"] + return True + + def prepare_template(tpl: dict[str, Any]) -> dict[str, Any] | None: """Prepare a template for use. @@ -226,6 +259,9 @@ def prepare_template(tpl: dict[str, Any]) -> dict[str, Any] | None: Returns: dict[str, Any] | None: Processed template dictionary. """ + if not _prepare_match_selector(tpl): + return None + # Test if all required fields are in template if "keywords" not in tpl: logger.warning( diff --git a/tests/test_page_ranges.py b/tests/test_page_ranges.py index 91d9795d..2e49c7a9 100644 --- a/tests/test_page_ranges.py +++ b/tests/test_page_ranges.py @@ -9,6 +9,7 @@ from invoice2data.api import _match_template_for_reader from invoice2data.extract.invoice_template import InvoiceTemplate +from invoice2data.extract.loader import prepare_template def _page_reader() -> types.ModuleType: @@ -143,3 +144,81 @@ def test_page_scoped_template_skips_unsupported_reader( assert selected is None assert "cannot use pages" in caplog.text + + +def test_selector_scoped_pages_can_match_and_extract_different_pages( + tmp_path: Path, +) -> None: + """Matching, scalar fields and modern lines each own their page scope.""" + pdf = tmp_path / "envelope-and-invoice.pdf" + pdf.write_bytes(b"%PDF-1.4") + reader = _page_reader() + + def to_text( + _path: str, + area_details: dict[str, Any] | None = None, + pages: tuple[int, int] | None = None, + ) -> str: + _ = area_details + content = { + None: "ENVELOPE SUPPLIER REF: G1\nINVOICE: INV-42\nLINE Widget", + (1, 1): "ENVELOPE SUPPLIER REF: G1", + (2, 3): "INVOICE: INV-42\nLINES START\nLINE Widget\nLINES END", + } + return content[pages] + + reader.to_text = to_text # type: ignore[attr-defined] + prepared = prepare_template( + { + "template_name": "supplier.yml", + "match": {"pages": 1, "keywords": ["ENVELOPE", "SUPPLIER"]}, + "required_fields": [], + "fields": { + "invoice_number": { + "parser": "regex", + "pages": "2-3", + "regex": r"INVOICE: (\S+)", + }, + "payment_reference": { + "parser": "regex", + "pages": 1, + "regex": r"REF: (\S+)", + }, + "lines": { + "parser": "lines", + "pages": "2-3", + "start": "LINES START", + "end": "LINES END", + "line": r"LINE (?P\w+)", + }, + }, + } + ) + assert prepared is not None + template = InvoiceTemplate(prepared) + + selected, document_text = _match_template_for_reader( + to_text(str(pdf)), [template], str(pdf), reader + ) + + assert selected is template + assert document_text.startswith("ENVELOPE") + result = template.extract(document_text, str(pdf), reader) + assert result["invoice_number"] == "INV-42" + assert result["payment_reference"] == "G1" + assert result["lines"] == [{"name": "Widget"}] + + +def test_match_selector_rejects_ambiguous_legacy_keywords() -> None: + """A template chooses either the explicit match selector or legacy keys.""" + assert ( + prepare_template( + { + "template_name": "ambiguous.yml", + "keywords": "legacy", + "pages": 1, + "match": {"keywords": "modern"}, + } + ) + is None + )