diff --git a/.env.example b/.env.example index 8097d6a..6a902b0 100644 --- a/.env.example +++ b/.env.example @@ -46,7 +46,19 @@ FOWOCO_INTENT_DEVICE=cpu FOWOCO_HF_TOKEN=hf_YOUR_HUGGINGFACE_TOKEN_HERE # ------------------------------------------------------------------------------ -# 5. CLOVA Template OCR — Stateless 내부 API +# 5. Dynamic Document Automation — local Qwen3 field mapping (opt-in) +# ------------------------------------------------------------------------------ +# false이면 model package/weights를 import 또는 load하지 않음 +FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED=false + +# 미설정 시 FOWOCO_MODEL_CACHE_DIR 아래의 pinned revision 경로를 사용 +# FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH=/data/model-cache/qwen3-embedding-0.6b/97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3 +# FOWOCO_DYNAMIC_AUTOMATION_RERANKER_MODEL_PATH=/data/model-cache/qwen3-reranker-0.6b/e61197ed45024b0ed8a2d74b80b4d909f1255473 +FOWOCO_DYNAMIC_AUTOMATION_MIN_RERANKER_SCORE=0.90 +FOWOCO_DYNAMIC_AUTOMATION_MIN_MARGIN=0.10 + +# ------------------------------------------------------------------------------ +# 6. CLOVA Template OCR — Stateless 내부 API # ------------------------------------------------------------------------------ # false이면 OCR API가 503을 반환하며 외부 HTTP client를 생성하지 않음 FOWOCO_CLOVA_OCR_ENABLED=false diff --git a/.superpowers/sdd/2026-08-11-dynamic-field-mapping-foundation/final-fix-report.md b/.superpowers/sdd/2026-08-11-dynamic-field-mapping-foundation/final-fix-report.md new file mode 100644 index 0000000..f81aaaf --- /dev/null +++ b/.superpowers/sdd/2026-08-11-dynamic-field-mapping-foundation/final-fix-report.md @@ -0,0 +1,202 @@ +# Dynamic Field Mapping Foundation — final fix report + +Date: 2026-08-11 + +Base commit: `d788f6e` + +Scope: the single user-authorized final fix wave. Existing registered-template/editing/HWP/HWPX/workflow behavior was left unchanged. The repository-wide baseline's 57 unrelated Language/Qdrant/Compose failures were not changed. + +All model-backed tests below used protocol-complete deterministic fakes. No Qwen weights were downloaded or executed. + +## 1. Mandatory Qwen execution evidence + +**Root cause.** `scripts/evaluate_dynamic_mapping.py` read undocumented `FOWOCO_QWEN3_*` variables, discarded mapper evidence, and certified precision alone. Lazy backend failures were converted to expected `AMBIGUOUS` mappings, so exact-rule cases plus an expected ambiguous case could produce a false-green Qwen report with no successful inference. + +**RED.** A direct CLI reproduction using only documented `FOWOCO_DYNAMIC_AUTOMATION_*` settings exited 1 without a report because the evaluator still required the alternate variables. Supplying the alternate variables while replacing `SentenceTransformer` with a constructor that raised reproduced the false green: exit 0, `gate.passed=true`, precision 1.0, coverage 0.5, and zero successful model inference. The focused regression command was: + +```text +python -m pytest tests/documents/dynamic_automation/test_evaluation.py::test_qwen_cli_fails_closed_when_lazy_model_inference_never_succeeds tests/documents/dynamic_automation/test_evaluation.py::test_qwen_cli_uses_documented_settings_and_records_fake_backend_execution -q +``` + +Before the fix the missing-model assertion observed exit 0 instead of 2, and the documented-settings case could not construct the Qwen mapper. + +**Minimal fix.** The evaluator now builds `Settings`, honors the documented enable flag, model paths, score threshold, and margin, and wraps both ports with successful-call counters. The JSON report records embedding/reranker execution counts and semantic case counts. Qwen certification requires both ports to execute successfully and every explicitly semantic expected-match case to pass. The fixture now contains `semantic-company-contact`, which cannot exact-rule match. Fake injection is limited to complete backend contracts on the Python entry point. + +**GREEN.** The focused command above reports `2 passed`. The final rule CLI still exits 0 with six cases, precision 1.0, sensitive precision 1.0, and coverage 0.4. + +**Files changed.** `scripts/evaluate_dynamic_mapping.py`; `tests/documents/dynamic_automation/test_evaluation.py`; `tests/fixtures/dynamic_automation/mapping_cases.jsonl`; `tests/integration/dynamic_automation/test_qwen_mapping_smoke.py`. + +## 2. Repeated labels and structural containers + +**Root cause.** Registry rows, nearby labels, and repeated-label counts were keyed without a table/container identity. Equal coordinates from different tables contaminated one another. Candidate filtering also treated `repeat_index > 0` as ineligible for non-repeatable canonical definitions. Field IDs were truncated to 200 characters and duplicate IDs were accepted. + +**RED.** The focused container/repeat/identity regressions produced three failures: there was no preserved `container_id`, equal table coordinates shared row/nearby context, a repeated phone context lost candidates, and duplicate/oversized identities were not rejected. + +```text +python -m pytest tests/documents/dynamic_automation/test_field_context.py::test_equal_coordinates_in_different_tables_are_container_local tests/documents/dynamic_automation/test_field_context.py::test_repeated_worker_and_company_phone_contexts_keep_compatible_candidates tests/documents/dynamic_automation/test_field_context.py::test_registry_rejects_duplicate_and_oversized_field_identities -q +``` + +**Minimal fix.** The registry adapter validates bounded `field_id` and real `target_id`, derives the table container from MCP target paths, rejects duplicate IDs, and never truncates identity fields. Row grouping, nearby-label search, section inference inputs, and repeated-label counters are container-local. Candidate compatibility no longer filters on repeat index; global mapping validation remains the repeatability enforcement layer. Fixtures use all fields emitted by `RegistryField.model_dump()`. + +**GREEN.** `python -m pytest tests/documents/dynamic_automation/test_field_context.py tests/documents/dynamic_automation/test_catalog.py -q` reports `13 passed`. + +**Files changed.** `app/documents/dynamic_automation/field_context.py`; `app/documents/dynamic_automation/models.py`; `app/documents/dynamic_automation/catalog.py`; `tests/documents/dynamic_automation/test_field_context.py`; `tests/documents/dynamic_automation/test_catalog.py`; `tests/fixtures/dynamic_automation/integrated_application_registry.json`; `tests/fixtures/dynamic_automation/extension_application_registry.json`. + +## 3. Actual MCP field-type compatibility + +**Root cause.** The canonical catalog declared invented transport types such as `business_number`, `alien_registration_number`, `name`, and `textarea`, while MCP serializes exactly nine registry types. A real serialized `number` business/alien-registration field therefore produced only the unrelated wage candidate or no identifier candidate. + +**RED.** Loading an actual-shape integrated registry and calling `catalog.compatible()` for its two `number` identifiers showed `company.business_number` and `identity.alien_registration_number` missing. The focused contract test failed on both assertions. + +```text +python -m pytest tests/documents/dynamic_automation/test_field_context.py::test_actual_mcp_number_fields_keep_identifier_candidates tests/documents/dynamic_automation/test_field_context.py::test_registry_fixture_and_type_union_match_actual_mcp_contract -q +``` + +**Minimal fix.** A constrained local `RegistryFieldType` mirrors MCP's `amount`, `checkbox`, `checkbox_group`, `date`, `number`, `phone`, `placeholder`, `signature`, and `text` union. Catalog `compatible_field_types` uses that type; semantic concepts remain in `value_type`. The YAML uses only real transport types and includes `number` for both registration identifiers. The contract fixture validates through the actual `hwp_mcp.fields.RegistryField` model and asserts union parity. + +**GREEN.** The focused command reports `2 passed`; it is also covered by the 13-pass field-context/catalog run. + +**Files changed.** `app/documents/dynamic_automation/models.py`; `app/documents/dynamic_automation/field_context.py`; `app/documents/dynamic_automation/resources/canonical_fields.v1.yaml`; `tests/documents/dynamic_automation/test_field_context.py`; integrated/extension registry and mapping-case fixtures. + +## 4. Definition-based reranking + +**Root cause.** `Qwen3CandidateReranker` sent `(context, canonical_field_id)` to the backend. The model never saw entity, aliases, value type, or description, and unknown IDs were never resolved. + +**RED.** The two new adapter regressions failed against the old constructor/behavior: the resolver keyword was unsupported, and the backend would have accepted a bare unknown ID. + +```text +python -m pytest tests/documents/dynamic_automation/test_qwen_adapters.py::test_reranker_backend_receives_full_resolved_canonical_definitions tests/documents/dynamic_automation/test_qwen_adapters.py::test_reranker_rejects_unknown_candidate_before_backend_execution -q +``` + +**Minimal fix.** The Qwen adapter accepts a definition resolver without changing the generic mapper port. It resolves the complete candidate batch before backend execution, verifies resolver identity, formats full definitions, and raises a clear fail-closed error for unknown/mismatched candidates. Evaluator, runtime loader, and conditional smoke constructors pass `catalog.get`. + +**GREEN.** The focused command reports `2 passed`. Assertions prove the backend document contains canonical ID, entity, value type, aliases, and description, and that an unknown candidate leaves `backend.pairs == []`. + +**Files changed.** `app/documents/dynamic_automation/qwen.py`; `app/documents/dynamic_automation/domain_adapters.py`; `scripts/evaluate_dynamic_mapping.py`; `scripts/train_dynamic_mapping_models.py`; `tests/documents/dynamic_automation/test_qwen_adapters.py`; `tests/integration/dynamic_automation/test_qwen_mapping_smoke.py`. + +## 5. Conservative promotion trade-off + +**Root cause.** The promotion predicate used coverage improvement **or** latency improvement. It promoted catastrophic loss on one axis when the other improved slightly. + +**RED.** Two direct regressions both observed `promote=True`: coverage `0.80 -> 0.01` with latency `200 -> 199`, and coverage `0.80 -> 0.81` with latency `200 -> 999`. + +```text +python -m pytest tests/documents/dynamic_automation/test_model_promotion.py::test_model_is_not_promoted_for_catastrophic_coverage_loss_with_latency_gain tests/documents/dynamic_automation/test_model_promotion.py::test_model_is_not_promoted_for_latency_regression_with_coverage_gain -q +``` + +**Minimal fix.** Promotion now requires coverage `>=` baseline, p95 latency `<=` baseline, and at least one strict improvement. Failures receive separate `coverage`, `p95_latency_ms`, or no-strict-improvement reasons. + +**GREEN.** `python -m pytest tests/documents/dynamic_automation/test_model_promotion.py -o addopts='' -q` reports `19 passed` after the final evidence-byte hardening. + +**Files changed.** `app/documents/dynamic_automation/training.py`; `tests/documents/dynamic_automation/test_model_promotion.py`. + +## 6. Evidence-bound, loadable domain adapters + +**Root cause.** Training wrote an unused bias/calibration JSON and a hand-filled manifest. No checked-in runtime loader applied the weights. Manifests had no exact artifact/report hashes or code/count bindings; comparison trusted manifest metrics and an editable unseen-field boolean. Pair calibration also discarded score orientation, so a reversed base ranker could not be corrected. + +**RED.** The initial end-to-end test failed at collection with `ModuleNotFoundError: app.documents.dynamic_automation.domain_adapters`; after introducing the interface, the old train entry point rejected injected complete backends. A comparison regression also showed that omitting all four exact artifact/report byte hashes still returned `PromotionDecision(promote=True)`. A separate final calibration RED reproduced the orientation defect: + +```text +python -m pytest tests/documents/dynamic_automation/test_domain_adapters.py::test_trained_reranker_calibration_can_reverse_a_wrong_base_ranking -q +``` + +It failed with `assert 1.25 < 0`, proving the learned head remained monotonic in the wrong direction. + +**Minimal fix.** `domain_adapters.py` provides strict typed v2 artifact loaders through the existing retriever/reranker ports. Query projection is applied to fresh base query vectors; reranker scale/bias is applied to fresh base scores. Loaders validate exact SHA-256, artifact kind/format, pinned repo/revision, dimensions, finiteness, and score contracts. Training exports the artifact first, reloads it through the public loader, evaluates held-out and generated unseen cases, hashes exact report bytes, and emits a manifest bound to artifact/report/dataset/catalog hashes, schema/code versions, pinned base, sample/cohort counts, and structured unseen candidate/rank evidence. Comparison requires all four actual artifact/report byte hashes, validates report cross-links, and derives gates from the report; omitted hashes fail closed. The calibration scale now preserves learned score orientation. + +**GREEN.** `python -m pytest tests/documents/dynamic_automation/test_domain_adapters.py -q` reports `4 passed`. The cache-independent CLI boundary command selecting the fake train/export/load/evaluate/compare test and the reversed-ranker test reports `2 passed`. Report or artifact tampering fails closed. + +**Files changed.** `app/documents/dynamic_automation/domain_adapters.py`; `app/documents/dynamic_automation/training.py`; `scripts/train_dynamic_mapping_models.py`; `scripts/compare_dynamic_mapping_models.py`; `tests/documents/dynamic_automation/test_domain_adapters.py`; `tests/documents/dynamic_automation/test_model_promotion.py`; `tests/documents/dynamic_automation/test_training_dataset.py`. + +## 7. Complete group-safe metadata + +**Root cause.** Sanitized feedback carried only `layout_hash`, and splitting grouped only that value. Forms sharing kind, version, or institution could cross train/test; pairwise grouping alone also missed transitive bridges. + +**RED.** Full v2 records with the required structural fields were rejected as an invalid schema/extra inputs, and a bridge `A(kind)=B; B(institution)=C` could cross partitions. + +```text +python -m pytest tests/documents/dynamic_automation/test_feedback.py::test_feedback_requires_bounded_group_metadata tests/documents/dynamic_automation/test_training_dataset.py::test_all_required_group_identities_are_disjoint_across_split tests/documents/dynamic_automation/test_training_dataset.py::test_training_split_keeps_transitively_connected_groups_together -q +``` + +The parametrized focused run produced five failing cases before the fix. + +**Minimal fix.** Feedback schema v2 requires bounded, nonempty `document_kind`, `document_version`, and `source_institution`; `from_review`, fixtures, ingestion, training examples, and dataset hashes carry them as value-free structural metadata. Deterministic union-find builds connected components sharing any required identity and splits whole components, so transitive relationships cannot leak. + +**GREEN.** The focused command reports `5 passed`; reversed input produces the identical split, and every required identity set is disjoint across train/test. + +**Files changed.** `app/documents/dynamic_automation/feedback.py`; `app/documents/dynamic_automation/training.py`; `scripts/train_dynamic_mapping_models.py`; `tests/documents/dynamic_automation/test_feedback.py`; `tests/documents/dynamic_automation/test_training_dataset.py`; `tests/fixtures/dynamic_automation/approved_feedback.jsonl`; `tests/documents/dynamic_automation/test_domain_adapters.py`. + +## 8. Reproducible dependency lock + +**Root cause.** `uv` was absent from the development interpreter and the committed lock did not represent the `document-automation` extra in `pyproject.toml`. + +**RED.** After installing `uv` only into the development environment, `python -m uv lock --check` exited nonzero and reported that `uv.lock` needed an update. + +**Minimal fix.** `python -m uv lock` regenerated only `uv.lock`; no unrelated dependency was added to `pyproject.toml`. + +**GREEN.** Both commands exit 0: + +```text +python -m uv lock --check +python -m uv sync --frozen --extra document-automation --dry-run +``` + +The frozen dry run resolves 136 packages and would install 74 packages. Because it was a dry run, it created no environment and downloaded neither packages nor model weights. The only environmental message was the existing Windows `SSL_CERT_DIR` certificate warning. + +**Files changed.** `uv.lock`. + +## 9. Immutable public catalog iteration + +**Root cause.** The frozen catalog dataclass held a mutable private dictionary. `catalog._fields_by_id.clear()` succeeded, and new training code reached into that private lookup; there was no stable public iteration contract. + +**RED.** The new catalog regression failed because `definitions` did not exist; the direct mutation reproduction removed all definitions from the supposedly frozen catalog. + +```text +python -m pytest tests/documents/dynamic_automation/test_catalog.py::test_catalog_definitions_are_immutable_and_stably_iterable -q +``` + +**Minimal fix.** The catalog's canonical public storage is a lexically sorted tuple of frozen definitions, `__iter__` exposes the same deterministic order, and the private ID lookup is a `MappingProxyType`. Training consumes `catalog.definitions`; production private-lookup references remain only inside `catalog.py`. + +**GREEN.** The focused command reports `1 passed`; tuple/definition/lookup mutation attempts fail and iteration order is stable. + +**Files changed.** `app/documents/dynamic_automation/catalog.py`; `app/documents/dynamic_automation/training.py`; `scripts/train_dynamic_mapping_models.py`; `tests/documents/dynamic_automation/test_catalog.py`. + +## Final verification + +```text +PYTHONUTF8=1 python -m pytest tests/documents/dynamic_automation -o addopts='' -q +134 passed in 11.24s + +python -m pytest tests/documents/dynamic_automation/test_model_promotion.py -o addopts='' -q +19 passed in 1.62s (run after the final exact-byte-hash hardening) + +PYTHONUTF8=1 python -m pytest tests/agents/test_document_field_map.py tests/agents/language/test_model_cache.py tests/documents/dynamic_automation/test_mapping_config.py tests/integration/language/test_compose_config.py -o addopts='' -q +36 passed in 6.65s + +python scripts/evaluate_dynamic_mapping.py --cases tests/fixtures/dynamic_automation/mapping_cases.jsonl --catalog app/documents/dynamic_automation/resources/canonical_fields.v1.yaml --mode rule --output +exit 0; 6 cases; precision 1.0; sensitive precision 1.0; coverage 0.4; gate passed + +PYTHONUTF8=1 python -m pytest tests/documents/dynamic_automation/test_domain_adapters.py::test_fake_backend_train_export_load_evaluate_and_compare_cli tests/documents/dynamic_automation/test_domain_adapters.py::test_trained_reranker_calibration_can_reverse_a_wrong_base_ranking -o addopts='' -q +2 passed in 0.54s + +python -m ruff check app/documents/dynamic_automation tests/documents/dynamic_automation scripts/evaluate_dynamic_mapping.py scripts/train_dynamic_mapping_models.py scripts/compare_dynamic_mapping_models.py +All checks passed! + +python -m uv lock --check +exit 0; resolved 136 packages + +python -m uv sync --frozen --extra document-automation --dry-run +exit 0; frozen resolution valid; would install 74 packages; no download performed + +git diff --check +exit 0 +``` + +Conditional real-model smoke: + +```text +PYTHONUTF8=1 python -m pytest tests/integration/dynamic_automation/test_qwen_mapping_smoke.py -o addopts='' -q +1 skipped in 0.16s +``` + +The skip is expected because both pinned Qwen caches are absent. No real model weights were downloaded. No additional concerns remain inside this fix wave; the known 57 unrelated Language/Qdrant/Compose baseline failures remain outside scope. diff --git a/app/documents/dynamic_automation/__init__.py b/app/documents/dynamic_automation/__init__.py new file mode 100644 index 0000000..c1f965c --- /dev/null +++ b/app/documents/dynamic_automation/__init__.py @@ -0,0 +1,25 @@ +"""Contracts and catalog data for additive dynamic document automation.""" + +from .catalog import CanonicalCatalog +from .models import ( + CanonicalFieldDefinition, + CanonicalMappingPlan, + CanonicalSource, + DocumentFieldContext, + FieldMapping, + MappingEvidence, + MappingStatus, + ScoredCandidate, +) + +__all__ = [ + "CanonicalCatalog", + "CanonicalFieldDefinition", + "CanonicalMappingPlan", + "CanonicalSource", + "DocumentFieldContext", + "FieldMapping", + "MappingEvidence", + "MappingStatus", + "ScoredCandidate", +] diff --git a/app/documents/dynamic_automation/catalog.py b/app/documents/dynamic_automation/catalog.py new file mode 100644 index 0000000..bc149e0 --- /dev/null +++ b/app/documents/dynamic_automation/catalog.py @@ -0,0 +1,77 @@ +"""Versioned canonical field catalog with allowlisted identifiers.""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from pathlib import Path +from types import MappingProxyType + +import yaml +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from .models import CanonicalFieldDefinition, DocumentFieldContext + + +class _CatalogDocument(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + version: str = Field(pattern=r"^v[1-9][0-9]*$") + fields: tuple[CanonicalFieldDefinition, ...] + + +@dataclass(frozen=True) +class CanonicalCatalog: + version: str + definitions: tuple[CanonicalFieldDefinition, ...] + _fields_by_id: Mapping[str, CanonicalFieldDefinition] = dataclass_field( + repr=False, + compare=False, + ) + + @classmethod + def load(cls, path: Path) -> CanonicalCatalog: + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as error: + raise ValueError(f"invalid canonical catalog: {error}") from error + + try: + document = _CatalogDocument.model_validate(raw) + except ValidationError as error: + if any("string_pattern_mismatch" == item["type"] for item in error.errors()): + raise ValueError("catalog contains an unapproved identifier") from error + raise ValueError(f"invalid canonical catalog: {error}") from error + + fields_by_id: dict[str, CanonicalFieldDefinition] = {} + for field in document.fields: + if field.field_id in fields_by_id: + raise ValueError(f"duplicate canonical field identifier: {field.field_id}") + fields_by_id[field.field_id] = field + + definitions = tuple(sorted(fields_by_id.values(), key=lambda item: item.field_id)) + immutable_lookup = MappingProxyType( + {definition.field_id: definition for definition in definitions} + ) + return cls( + version=document.version, + definitions=definitions, + _fields_by_id=immutable_lookup, + ) + + def __iter__(self) -> Iterator[CanonicalFieldDefinition]: + return iter(self.definitions) + + def get(self, field_id: str) -> CanonicalFieldDefinition: + try: + return self._fields_by_id[field_id] + except KeyError as error: + raise KeyError(f"unknown canonical field: {field_id}") from error + + def compatible(self, context: DocumentFieldContext) -> tuple[CanonicalFieldDefinition, ...]: + return tuple( + field + for field in self.definitions + if context.field_type in field.compatible_field_types + ) diff --git a/app/documents/dynamic_automation/config.py b/app/documents/dynamic_automation/config.py new file mode 100644 index 0000000..7176d03 --- /dev/null +++ b/app/documents/dynamic_automation/config.py @@ -0,0 +1,84 @@ +"""Package-owned settings for opt-in dynamic document mapping.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path +from typing import Self + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +from .qwen import ( + QWEN3_EMBEDDING_CACHE_NAME, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_CACHE_NAME, + QWEN3_RERANKER_REVISION, +) + +_EMBEDDING_PATH = Path(QWEN3_EMBEDDING_CACHE_NAME) / QWEN3_EMBEDDING_REVISION +_RERANKER_PATH = Path(QWEN3_RERANKER_CACHE_NAME) / QWEN3_RERANKER_REVISION + + +class DynamicAutomationSettings(BaseSettings): + """Read dynamic-only settings without extending the FastAPI settings model.""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + env_prefix="FOWOCO_", + extra="ignore", + ) + + model_cache_dir: Path = Field( + default_factory=lambda: Path(tempfile.gettempdir()) / "fowoco-model-cache" + ) + dynamic_automation_mapping_enabled: bool = False + dynamic_automation_embedding_model_path: Path | None = None + dynamic_automation_reranker_model_path: Path | None = None + dynamic_automation_min_reranker_score: float = Field(default=0.90, ge=0, le=1) + dynamic_automation_min_margin: float = Field(default=0.10, ge=0, le=1) + + @model_validator(mode="after") + def derive_model_paths(self) -> Self: + """Resolve and validate pinned model paths inside the managed cache.""" + embedding_path = self.dynamic_automation_embedding_model_path or ( + self.model_cache_dir / _EMBEDDING_PATH + ) + reranker_path = self.dynamic_automation_reranker_model_path or ( + self.model_cache_dir / _RERANKER_PATH + ) + self.dynamic_automation_embedding_model_path = _managed_model_path( + embedding_path, + model_cache_dir=self.model_cache_dir, + pinned_suffix=_EMBEDDING_PATH, + setting_name="dynamic_automation_embedding_model_path", + ) + self.dynamic_automation_reranker_model_path = _managed_model_path( + reranker_path, + model_cache_dir=self.model_cache_dir, + pinned_suffix=_RERANKER_PATH, + setting_name="dynamic_automation_reranker_model_path", + ) + return self + + +def _managed_model_path( + path: Path, + *, + model_cache_dir: Path, + pinned_suffix: Path, + setting_name: str, +) -> Path: + resolved_cache = model_cache_dir.resolve(strict=False) + resolved_path = path.resolve(strict=False) + try: + relative_path = resolved_path.relative_to(resolved_cache) + except ValueError as err: + raise ValueError(f"{setting_name} must be below model_cache_dir") from err + if relative_path.parts[-2:] != pinned_suffix.parts: + raise ValueError( + f"{setting_name} must end in the pinned revision directory " + f"{pinned_suffix.as_posix()}" + ) + return resolved_path diff --git a/app/documents/dynamic_automation/domain_adapters.py b/app/documents/dynamic_automation/domain_adapters.py new file mode 100644 index 0000000..5dfa149 --- /dev/null +++ b/app/documents/dynamic_automation/domain_adapters.py @@ -0,0 +1,317 @@ +"""Load deterministic domain heads over fixed local Qwen mapping backends.""" + +from __future__ import annotations + +import hashlib +import math +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator + +from .models import CanonicalFieldDefinition, DocumentFieldContext, ScoredCandidate +from .qwen import ( + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_REPO, + QWEN3_RERANKER_REVISION, + EmbeddingBackend, + Qwen3CandidateReranker, + Qwen3EmbeddingRetriever, + RerankerBackend, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class QueryBiasProjection(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + adapter_kind: Literal["query_bias_projection"] + embedding_dimension: int = Field(ge=1, le=100_000) + positive_pair_count: int = Field(ge=1) + query_bias: tuple[float, ...] = Field(min_length=1, max_length=100_000) + + @model_validator(mode="after") + def _dimension_matches_weights(self) -> QueryBiasProjection: + if len(self.query_bias) != self.embedding_dimension: + raise ValueError("query projection dimension does not match query_bias") + if not all(math.isfinite(value) for value in self.query_bias): + raise ValueError("query projection weights must be finite") + return self + + +class ScoreCalibration(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + adapter_kind: Literal["score_calibration"] + positive_pair_count: int = Field(ge=1) + negative_pair_count: int = Field(ge=1) + scale: float + bias: float + + @model_validator(mode="after") + def _weights_are_finite(self) -> ScoreCalibration: + if not math.isfinite(self.scale) or not math.isfinite(self.bias): + raise ValueError("reranker calibration weights must be finite") + return self + + +AdapterWeights = Annotated[ + QueryBiasProjection | ScoreCalibration, + Field(discriminator="adapter_kind"), +] + + +class DomainAdapterArtifact(BaseModel): + """Strict, portable runtime artifact produced by the training command.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + format_version: Literal["dynamic-mapping-adapter-v2"] + model_kind: Literal["bi-encoder", "pair-reranker"] + base_model_repo: str = Field(min_length=1, max_length=300) + base_model_revision: str = Field(min_length=1, max_length=200) + seed: int = Field(ge=0) + weights: AdapterWeights + + @model_validator(mode="after") + def _kind_matches_weights_and_pinned_base(self) -> DomainAdapterArtifact: + if self.model_kind == "bi-encoder": + if not isinstance(self.weights, QueryBiasProjection): + raise ValueError("bi-encoder artifact requires query projection weights") + expected = (QWEN3_EMBEDDING_REPO, QWEN3_EMBEDDING_REVISION) + else: + if not isinstance(self.weights, ScoreCalibration): + raise ValueError("pair-reranker artifact requires calibration weights") + expected = (QWEN3_RERANKER_REPO, QWEN3_RERANKER_REVISION) + if (self.base_model_repo, self.base_model_revision) != expected: + raise ValueError("domain adapter does not identify the pinned Qwen base") + return self + + +@dataclass(frozen=True) +class DomainEmbeddingRetriever: + delegate: Qwen3EmbeddingRetriever + artifact_sha256: str + + @property + def model_version(self) -> str: + return f"domain-bi-encoder@{self.artifact_sha256}" + + def retrieve( + self, + context: DocumentFieldContext, + candidates: Sequence[CanonicalFieldDefinition], + top_k: int, + ) -> tuple[ScoredCandidate, ...]: + return self.delegate.retrieve(context, candidates, top_k) + + +@dataclass(frozen=True) +class DomainCandidateReranker: + delegate: Qwen3CandidateReranker + artifact_sha256: str + + @property + def model_version(self) -> str: + return f"domain-pair-reranker@{self.artifact_sha256}" + + def rerank( + self, + context: DocumentFieldContext, + candidates: Sequence[ScoredCandidate], + ) -> tuple[ScoredCandidate, ...]: + return self.delegate.rerank(context, candidates) + + +def adapter_file_sha256(path: str | Path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def load_domain_embedding_retriever( + artifact_path: str | Path, + *, + backend: EmbeddingBackend | None = None, + model_path: str | Path | None = None, + expected_sha256: str | None = None, + max_length: int = 512, + batch_size: int = 8, +) -> DomainEmbeddingRetriever: + artifact, digest = _load_artifact(artifact_path, expected_sha256=expected_sha256) + if artifact.model_kind != "bi-encoder" or not isinstance( + artifact.weights, QueryBiasProjection + ): + raise ValueError("artifact is not a domain bi-encoder projection") + base = Qwen3EmbeddingRetriever( + model_path, + backend=backend, + max_length=max_length, + batch_size=batch_size, + ) + projected = Qwen3EmbeddingRetriever( + backend=_ProjectedEmbeddingBackend(base.backend, artifact.weights), + max_length=max_length, + batch_size=batch_size, + ) + return DomainEmbeddingRetriever(projected, digest) + + +def load_domain_reranker( + artifact_path: str | Path, + *, + definition_resolver: Callable[[str], CanonicalFieldDefinition], + backend: RerankerBackend | None = None, + model_path: str | Path | None = None, + expected_sha256: str | None = None, + max_length: int = 512, + batch_size: int = 8, +) -> DomainCandidateReranker: + artifact, digest = _load_artifact(artifact_path, expected_sha256=expected_sha256) + if artifact.model_kind != "pair-reranker" or not isinstance( + artifact.weights, ScoreCalibration + ): + raise ValueError("artifact is not a domain pair-reranker calibration") + base = Qwen3CandidateReranker( + model_path, + backend=backend, + definition_resolver=definition_resolver, + max_length=max_length, + batch_size=batch_size, + ) + calibrated = Qwen3CandidateReranker( + backend=_CalibratedRerankerBackend(base.backend, artifact.weights), + definition_resolver=definition_resolver, + max_length=max_length, + batch_size=batch_size, + ) + return DomainCandidateReranker(calibrated, digest) + + +def _load_artifact( + path: str | Path, *, expected_sha256: str | None +) -> tuple[DomainAdapterArtifact, str]: + artifact_path = Path(path) + payload = artifact_path.read_bytes() + digest = hashlib.sha256(payload).hexdigest() + if expected_sha256 is not None: + if not _is_sha256(expected_sha256) or digest != expected_sha256: + raise ValueError("domain adapter SHA-256 does not match the manifest") + try: + artifact = DomainAdapterArtifact.model_validate_json(payload) + except ValidationError as error: + raise ValueError(f"invalid domain adapter artifact: {error}") from error + return artifact, digest + + +@dataclass(frozen=True) +class _ProjectedEmbeddingBackend: + base: EmbeddingBackend + projection: QueryBiasProjection + + def encode_queries( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + vectors = self.base.encode_queries( + texts, max_length=max_length, batch_size=batch_size + ) + validated = _validate_vectors(vectors, self.projection.embedding_dimension) + return tuple( + _normalize( + tuple( + value + bias + for value, bias in zip( + vector, self.projection.query_bias, strict=True + ) + ) + ) + for vector in validated + ) + + def encode_documents( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + vectors = self.base.encode_documents( + texts, max_length=max_length, batch_size=batch_size + ) + return _validate_vectors(vectors, self.projection.embedding_dimension) + + +@dataclass(frozen=True) +class _CalibratedRerankerBackend: + base: RerankerBackend + calibration: ScoreCalibration + + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: + raw_scores = self.base.score_pairs( + pairs, max_length=max_length, batch_size=batch_size + ) + if len(raw_scores) != len(pairs): + raise RuntimeError("base reranker returned the wrong score count") + calibrated: list[float] = [] + for raw_score in raw_scores: + score = float(raw_score) + if not math.isfinite(score) or not 0 <= score <= 1: + raise RuntimeError("base reranker returned an invalid probability") + calibrated.append( + _sigmoid(self.calibration.scale * score + self.calibration.bias) + ) + return tuple(calibrated) + + +def _validate_vectors( + vectors: Sequence[Sequence[float]], dimension: int +) -> tuple[tuple[float, ...], ...]: + validated = tuple(tuple(float(value) for value in vector) for vector in vectors) + if any(len(vector) != dimension for vector in validated): + raise RuntimeError("base embedding dimension does not match the adapter") + if any(not math.isfinite(value) for vector in validated for value in vector): + raise RuntimeError("base embedding returned non-finite values") + return validated + + +def _normalize(vector: tuple[float, ...]) -> tuple[float, ...]: + norm = math.sqrt(sum(value * value for value in vector)) + if norm == 0: + return vector + return tuple(value / norm for value in vector) + + +def _sigmoid(value: float) -> float: + if value >= 0: + exponential = math.exp(-value) + return 1 / (1 + exponential) + exponential = math.exp(value) + return exponential / (1 + exponential) + + +def _is_sha256(value: str) -> bool: + if len(value) != 64: + return False + return all(character in "0123456789abcdef" for character in value) + + +__all__ = [ + "DomainAdapterArtifact", + "DomainCandidateReranker", + "DomainEmbeddingRetriever", + "adapter_file_sha256", + "load_domain_embedding_retriever", + "load_domain_reranker", +] diff --git a/app/documents/dynamic_automation/feedback.py b/app/documents/dynamic_automation/feedback.py new file mode 100644 index 0000000..3e1587b --- /dev/null +++ b/app/documents/dynamic_automation/feedback.py @@ -0,0 +1,173 @@ +"""Privacy-safe, append-only reviewer feedback for mapping decisions.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .models import CanonicalMappingPlan, DocumentFieldContext, MappingStatus + +_CANONICAL_ID_PATTERN = r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$" +_HASH_PATTERN = r"^[0-9a-f]{64}$" +_FORBIDDEN_KEY = re.compile( + r"(?:^|[_.-])(?:value|passport|registration_number|resident_number)(?:$|[_.-])", + flags=re.IGNORECASE, +) + + +class ReviewerDecision(StrEnum): + ACCEPTED = "accepted" + CORRECTED = "corrected" + REJECTED = "rejected" + + +class FeedbackCandidateScore(BaseModel): + """Bounded candidate evidence safe for feedback persistence.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + canonical_field_id: str = Field(max_length=200, pattern=_CANONICAL_ID_PATTERN) + score: float = Field(ge=0, le=1) + rank: int = Field(ge=1) + + +class MappingFeedbackRecord(BaseModel): + """Value-free metadata for one reviewed field-mapping decision.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["v2"] = "v2" + layout_hash: str = Field(pattern=_HASH_PATTERN) + document_kind: str = Field(min_length=1, max_length=100) + document_version: str = Field(min_length=1, max_length=100) + source_institution: str = Field(min_length=1, max_length=100) + field_context_hash: str = Field(pattern=_HASH_PATTERN) + field_id: str = Field(min_length=1, max_length=200) + repeat_index: int = Field(ge=0) + label: str = Field(max_length=200) + section: str = Field(max_length=200) + row_labels: tuple[Annotated[str, Field(max_length=200)], ...] = Field(max_length=3) + nearby_labels: tuple[Annotated[str, Field(max_length=200)], ...] = Field(max_length=4) + predicted_status: MappingStatus + predicted_canonical_field_id: str | None = Field( + default=None, max_length=200, pattern=_CANONICAL_ID_PATTERN + ) + final_canonical_field_id: str | None = Field( + default=None, max_length=200, pattern=_CANONICAL_ID_PATTERN + ) + decision: ReviewerDecision + candidate_scores: tuple[FeedbackCandidateScore, ...] = Field(default=(), max_length=20) + catalog_version: str = Field(max_length=20, pattern=r"^v[1-9][0-9]*$") + model_version: str | None = Field(default=None, max_length=200) + + @model_validator(mode="before") + @classmethod + def _reject_sensitive_keys(cls, value: Any) -> Any: + _validate_keys(value) + return value + + @model_validator(mode="after") + def _validate_prediction(self) -> MappingFeedbackRecord: + has_id = self.predicted_canonical_field_id is not None + if (self.predicted_status is MappingStatus.MATCHED) != has_id: + raise ValueError("only matched predictions may include a canonical field ID") + return self + + @classmethod + def from_review( + cls, + plan: CanonicalMappingPlan, + context: DocumentFieldContext, + *, + layout_hash: str, + document_kind: str, + document_version: str, + source_institution: str, + decision: ReviewerDecision, + final_canonical_field_id: str | None, + ) -> MappingFeedbackRecord: + """Build one record from an existing mapping plan and reviewer decision.""" + matches = tuple( + mapping + for mapping in plan.mappings + if mapping.field_id == context.field_id + and mapping.repeat_index == context.repeat_index + ) + if len(matches) != 1: + raise ValueError("mapping plan must contain exactly one matching field context") + mapping = matches[0] + return cls( + layout_hash=layout_hash, + document_kind=document_kind, + document_version=document_version, + source_institution=source_institution, + field_context_hash=hash_field_context(context), + field_id=context.field_id, + repeat_index=context.repeat_index, + label=context.label, + section=context.section, + row_labels=context.row_labels, + nearby_labels=context.nearby_labels, + predicted_status=mapping.status, + predicted_canonical_field_id=mapping.canonical_field_id, + final_canonical_field_id=final_canonical_field_id, + decision=decision, + candidate_scores=tuple( + FeedbackCandidateScore.model_validate(candidate.model_dump()) + for candidate in mapping.candidates + ), + catalog_version=plan.catalog_version, + model_version=mapping.evidence.model_version, + ) + +class JsonlMappingFeedbackStore: + """Append validated feedback records without providing update or delete operations.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + + def append(self, record: MappingFeedbackRecord) -> None: + if not isinstance(record, MappingFeedbackRecord): + raise TypeError("record must be a MappingFeedbackRecord") + validated = MappingFeedbackRecord.model_validate( + record.model_dump(mode="json", warnings="none") + ) + serialized = json.dumps( + validated.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as stream: + stream.write(serialized) + stream.write("\n") + + +def hash_field_context(context: DocumentFieldContext) -> str: + """Return a deterministic hash without retaining an unbounded source payload.""" + serialized = json.dumps( + context.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(serialized).hexdigest() + + +def _validate_keys(value: Any) -> None: + if isinstance(value, Mapping): + for key, nested in value.items(): + if not isinstance(key, str) or _FORBIDDEN_KEY.search(key): + raise ValueError(f"feedback contains a forbidden key: {key!r}") + _validate_keys(nested) + elif isinstance(value, (list, tuple)): + for nested in value: + _validate_keys(nested) diff --git a/app/documents/dynamic_automation/field_context.py b/app/documents/dynamic_automation/field_context.py new file mode 100644 index 0000000..47aa10e --- /dev/null +++ b/app/documents/dynamic_automation/field_context.py @@ -0,0 +1,135 @@ +"""Build bounded structural context from untrusted MCP field registries.""" + +from __future__ import annotations + +import re +import unicodedata +from collections import defaultdict +from collections.abc import Mapping, Sequence +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from .models import DocumentFieldContext, RegistryFieldType + +_MAX_TEXT_LENGTH = 200 +_MAX_OPTIONS = 50 + + +class _RegistryInput(BaseModel): + """Only the registry fields needed for value-free mapping context.""" + + model_config = ConfigDict(extra="ignore") + + field_id: str = Field(min_length=1, max_length=_MAX_TEXT_LENGTH) + target_id: str = Field(min_length=1, max_length=_MAX_TEXT_LENGTH) + label: str = "" + type: RegistryFieldType + kind: str = Field(min_length=1) + row: int + column: int + required: bool = True + options: tuple[str, ...] | None = None + + +def normalize_text(value: str) -> str: + """Normalize user-facing labels for deterministic equality checks.""" + normalized = unicodedata.normalize("NFKC", value).casefold() + return re.sub(r"[\W_]+", "", normalized, flags=re.UNICODE) + + +def build_field_contexts( + registry: Sequence[Mapping[str, Any]], *, document_title: str +) -> tuple[DocumentFieldContext, ...]: + """Return one bounded structural context per registry item.""" + validated = [_RegistryInput.model_validate(item) for item in registry] + field_ids = [item.field_id for item in validated] + if len(set(field_ids)) != len(field_ids): + raise ValueError("registry contains a duplicate field_id") + repeat_indices = _repeat_indices(validated) + row_groups = _row_groups(validated) + return tuple( + _context_for( + item, + validated, + row_groups=row_groups, + document_title=document_title, + repeat_index=repeat_indices[index], + ) + for index, item in enumerate(validated) + ) + + +def _context_for( + item: _RegistryInput, + registry: Sequence[_RegistryInput], + *, + row_groups: Mapping[tuple[str, int], tuple[_RegistryInput, ...]], + document_title: str, + repeat_index: int, +) -> DocumentFieldContext: + container_id = _container_id(item.target_id) + row_items = row_groups[(container_id, item.row)] + row_labels = tuple(_bound_text(candidate.label) for candidate in row_items[:3]) + nearby = sorted( + ( + candidate + for candidate in registry + if _container_id(candidate.target_id) == container_id + and candidate.row != item.row + ), + key=lambda candidate: ( + abs(candidate.row - item.row), + abs(candidate.column - item.column), + candidate.row, + candidate.column, + ), + ) + return DocumentFieldContext( + field_id=item.field_id, + container_id=container_id, + label=_bound_text(item.label), + normalized_label=_bound_text(normalize_text(item.label)), + field_type=_bound_text(item.type), + document_title=_bound_text(document_title), + section=row_labels[0] if row_labels else "", + row_labels=row_labels, + nearby_labels=tuple(_bound_text(candidate.label) for candidate in nearby[:4]), + options=tuple(_bound_text(option) for option in (item.options or ())[:_MAX_OPTIONS]), + repeat_index=repeat_index, + required=item.required, + kind=_bound_text(item.kind), + ) + + +def _row_groups( + registry: Sequence[_RegistryInput], +) -> dict[tuple[str, int], tuple[_RegistryInput, ...]]: + grouped: dict[tuple[str, int], list[tuple[int, _RegistryInput]]] = defaultdict(list) + for index, item in enumerate(registry): + grouped[(_container_id(item.target_id), item.row)].append((index, item)) + return { + group: tuple( + item for _, item in sorted(items, key=lambda pair: (pair[1].column, pair[0])) + ) + for group, items in grouped.items() + } + + +def _repeat_indices(registry: Sequence[_RegistryInput]) -> tuple[int, ...]: + counts: dict[tuple[str, str], int] = defaultdict(int) + indices: list[int] = [] + for item in registry: + key = (_container_id(item.target_id), normalize_text(item.label)) + indices.append(counts[key]) + counts[key] += 1 + return tuple(indices) + + +def _bound_text(value: str) -> str: + return value[:_MAX_TEXT_LENGTH] + + +def _container_id(target_id: str) -> str: + match = re.match(r"^(.*?\.table[^.]+)(?:\.|$)", target_id) + return match.group(1) if match is not None else target_id diff --git a/app/documents/dynamic_automation/global_validation.py b/app/documents/dynamic_automation/global_validation.py new file mode 100644 index 0000000..b64d801 --- /dev/null +++ b/app/documents/dynamic_automation/global_validation.py @@ -0,0 +1,93 @@ +"""Document-wide validation for independently mapped fields.""" + +from __future__ import annotations + +from collections import defaultdict + +from .catalog import CanonicalCatalog +from .models import CanonicalMappingPlan, FieldMapping, MappingStatus + + +def validate_global_mapping( + plan: CanonicalMappingPlan, catalog: CanonicalCatalog +) -> CanonicalMappingPlan: + """Downgrade globally conflicting matches without changing unresolved fields.""" + reasons: dict[int, str] = {} + matched_by_canonical: dict[str, list[tuple[int, FieldMapping]]] = defaultdict(list) + + for index, mapping in enumerate(plan.mappings): + if mapping.status is not MappingStatus.MATCHED: + continue + canonical_field_id = mapping.canonical_field_id + if canonical_field_id is None: + continue + if ( + plan.catalog_version != catalog.version + or mapping.evidence.catalog_version != catalog.version + ): + reasons[index] = "catalog_version_mismatch" + elif not _has_complete_match_evidence(mapping): + reasons[index] = "incomplete_mapping_evidence" + try: + definition = catalog.get(canonical_field_id) + except KeyError: + reasons[index] = "unknown_canonical_field" + continue + if ( + mapping.evidence.entity_hint is not None + and mapping.evidence.entity_hint != definition.entity + ): + reasons.setdefault(index, "incompatible_entity_role") + matched_by_canonical[canonical_field_id].append((index, mapping)) + + for canonical_field_id, indexed_mappings in matched_by_canonical.items(): + definition = catalog.get(canonical_field_id) + if not definition.repeatable and len(indexed_mappings) > 1: + for index, _ in indexed_mappings: + reasons.setdefault(index, "duplicate_non_repeatable_canonical_field") + continue + + by_repeat_index: dict[int, list[int]] = defaultdict(list) + for index, mapping in indexed_mappings: + by_repeat_index[mapping.repeat_index].append(index) + for indexes in by_repeat_index.values(): + if len(indexes) > 1: + for index in indexes: + reasons.setdefault(index, "duplicate_repeat_index") + + mappings = tuple( + _downgrade(mapping, reasons[index]) if index in reasons else mapping + for index, mapping in enumerate(plan.mappings) + ) + return plan.model_copy(update={"mappings": mappings}) + + +def _has_complete_match_evidence(mapping: FieldMapping) -> bool: + evidence = mapping.evidence + common_complete = evidence.type_compatible is True and bool(evidence.model_version) + if evidence.rule == "exact_alias": + return ( + common_complete + and evidence.reason == "exact_alias" + and evidence.entity_hint is not None + ) + if evidence.rule == "semantic_decision_gate": + return ( + common_complete + and evidence.reason == "decision_gate_passed" + and evidence.embedding_rank is not None + and evidence.reranker_score is not None + and evidence.score_margin is not None + ) + return False + + +def _downgrade(mapping: FieldMapping, reason: str) -> FieldMapping: + evidence = mapping.evidence.model_copy(update={"reason": reason}) + return mapping.model_copy( + update={ + "status": MappingStatus.AMBIGUOUS, + "canonical_field_id": None, + "evidence": evidence, + } + ) diff --git a/app/documents/dynamic_automation/mapper.py b/app/documents/dynamic_automation/mapper.py new file mode 100644 index 0000000..d9b61cd --- /dev/null +++ b/app/documents/dynamic_automation/mapper.py @@ -0,0 +1,334 @@ +"""Fail-closed hybrid canonical field mapping orchestration.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field + +from .catalog import CanonicalCatalog +from .field_context import normalize_text +from .global_validation import validate_global_mapping +from .models import ( + CanonicalFieldDefinition, + CanonicalMappingPlan, + DocumentFieldContext, + FieldMapping, + MappingEvidence, + MappingStatus, + ScoredCandidate, +) +from .ports import CandidateReranker, CandidateRetriever +from .rules import classify_non_data, exact_alias_matches + +_RULE_MODEL_VERSION = "deterministic-rules-v1" +_ENTITY_TERMS = { + "worker": ("근로자", "신청인", "worker", "employee", "applicant"), + "company": ("회사", "사업장", "근무처", "고용주", "company", "employer", "workplace"), + "identity": ("신원", "여권", "외국인등록", "identity", "passport"), + "contract": ("계약", "근로조건", "contract"), + "application": ("신청일", "application date"), +} + + +class MappingThresholds(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + min_reranker_score: float = Field(ge=0, le=1) + min_margin: float = Field(ge=0, le=1) + exact_alias_requires_unique_entity: bool = True + + +@dataclass(frozen=True) +class HybridFieldMapper: + catalog: CanonicalCatalog + retriever: CandidateRetriever + reranker: CandidateReranker + thresholds: MappingThresholds + top_k: int = 10 + + def __post_init__(self) -> None: + if self.top_k < 1: + raise ValueError("top_k must be positive") + + def map(self, contexts: Sequence[DocumentFieldContext]) -> CanonicalMappingPlan: + mappings = tuple(self._map_one(context) for context in contexts) + return validate_global_mapping( + CanonicalMappingPlan(catalog_version=self.catalog.version, mappings=mappings), + self.catalog, + ) + + def _map_one(self, context: DocumentFieldContext) -> FieldMapping: + entity_hint = _entity_hint(context) + non_data = classify_non_data(context) + if non_data.is_non_data: + return self._mapping( + context, + status=MappingStatus.NON_DATA, + reason=non_data.reason or "non_data", + rule="non_data_rule", + entity_hint=entity_hint, + model_version=_RULE_MODEL_VERSION, + ) + + compatible = self.catalog.compatible(context) + if not compatible: + return self._mapping( + context, + status=MappingStatus.UNMAPPED, + reason="no_compatible_candidates", + rule="candidate_filter", + type_compatible=False, + entity_hint=entity_hint, + model_version=_RULE_MODEL_VERSION, + ) + + exact = exact_alias_matches(context, self.catalog) + if _is_unique_exact_match( + exact, + entity_hint=entity_hint, + require_unique_entity=self.thresholds.exact_alias_requires_unique_entity, + ): + definition = exact[0] + candidate = ScoredCandidate( + canonical_field_id=definition.field_id, + score=1.0, + rank=1, + ) + return self._mapping( + context, + status=MappingStatus.MATCHED, + canonical_field_id=definition.field_id, + candidates=(candidate,), + reason="exact_alias", + rule="exact_alias", + type_compatible=True, + entity_hint=entity_hint or definition.entity, + model_version=_RULE_MODEL_VERSION, + ) + + try: + retrieved = self.retriever.retrieve(context, compatible, self.top_k) + except Exception: + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + reason="retriever_unavailable", + rule="semantic_decision_gate", + type_compatible=True, + entity_hint=entity_hint, + model_version=self.retriever.model_version, + ) + + if not isinstance(retrieved, tuple): + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + reason="invalid_retrieval_evidence", + rule="semantic_decision_gate", + entity_hint=entity_hint, + model_version=self.retriever.model_version, + ) + + allowed_ids = {candidate.field_id for candidate in compatible} + if len(retrieved) > self.top_k or not _valid_ranking( + retrieved, allowed_ids=allowed_ids, require_all_ids=False + ): + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + candidates=retrieved, + reason="invalid_retrieval_evidence", + rule="semantic_decision_gate", + type_compatible=all( + candidate.canonical_field_id in allowed_ids for candidate in retrieved + ), + entity_hint=entity_hint, + model_version=self.retriever.model_version, + ) + if not retrieved: + return self._mapping( + context, + status=MappingStatus.UNMAPPED, + reason="no_retrieval_candidates", + rule="semantic_decision_gate", + type_compatible=True, + entity_hint=entity_hint, + model_version=self.retriever.model_version, + ) + + try: + reranked = self.reranker.rerank(context, retrieved) + except Exception: + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + candidates=retrieved, + reason="reranker_unavailable", + rule="semantic_decision_gate", + embedding_rank=retrieved[0].rank, + type_compatible=True, + entity_hint=entity_hint, + model_version=self.reranker.model_version, + ) + + if not isinstance(reranked, tuple): + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + reason="invalid_reranker_evidence", + rule="semantic_decision_gate", + entity_hint=entity_hint, + model_version=self.reranker.model_version, + ) + + retrieved_ids = {candidate.canonical_field_id for candidate in retrieved} + if not _valid_ranking(reranked, allowed_ids=retrieved_ids, require_all_ids=True): + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + candidates=reranked, + reason="invalid_reranker_evidence", + rule="semantic_decision_gate", + type_compatible=all( + candidate.canonical_field_id in retrieved_ids for candidate in reranked + ), + entity_hint=entity_hint, + model_version=self.reranker.model_version, + ) + + top = reranked[0] + embedding_rank = next( + candidate.rank + for candidate in retrieved + if candidate.canonical_field_id == top.canonical_field_id + ) + if len(reranked) < 2: + return self._mapping( + context, + status=MappingStatus.AMBIGUOUS, + candidates=reranked, + reason="insufficient_margin_evidence", + rule="semantic_decision_gate", + embedding_rank=embedding_rank, + reranker_score=top.score, + type_compatible=True, + entity_hint=entity_hint, + model_version=self.reranker.model_version, + ) + + score_margin = top.score - reranked[1].score + reason = "decision_gate_passed" + status = MappingStatus.MATCHED + canonical_field_id: str | None = top.canonical_field_id + if top.score < self.thresholds.min_reranker_score: + reason = "insufficient_score" + status = MappingStatus.AMBIGUOUS + canonical_field_id = None + elif score_margin < self.thresholds.min_margin: + reason = "insufficient_margin" + status = MappingStatus.AMBIGUOUS + canonical_field_id = None + + return self._mapping( + context, + status=status, + canonical_field_id=canonical_field_id, + candidates=reranked, + reason=reason, + rule="semantic_decision_gate", + embedding_rank=embedding_rank, + reranker_score=top.score, + score_margin=score_margin, + type_compatible=True, + entity_hint=entity_hint, + model_version=self.reranker.model_version, + ) + + def _mapping( + self, + context: DocumentFieldContext, + *, + status: MappingStatus, + reason: str, + rule: str, + candidates: tuple[ScoredCandidate, ...] = (), + canonical_field_id: str | None = None, + embedding_rank: int | None = None, + reranker_score: float | None = None, + score_margin: float | None = None, + type_compatible: bool | None = None, + entity_hint: str | None = None, + model_version: str | None = None, + ) -> FieldMapping: + return FieldMapping( + field_id=context.field_id, + repeat_index=context.repeat_index, + status=status, + canonical_field_id=canonical_field_id, + candidates=candidates, + evidence=MappingEvidence( + reason=reason, + rule=rule, + embedding_rank=embedding_rank, + reranker_score=reranker_score, + score_margin=score_margin, + type_compatible=type_compatible, + entity_hint=entity_hint, + catalog_version=self.catalog.version, + model_version=model_version, + ), + ) + + +def _is_unique_exact_match( + matches: Sequence[CanonicalFieldDefinition], + *, + entity_hint: str | None, + require_unique_entity: bool, +) -> bool: + if len(matches) != 1: + return False + if entity_hint is not None and matches[0].entity != entity_hint: + return False + if require_unique_entity and entity_hint is None: + return False + return True + + +def _valid_ranking( + candidates: Sequence[ScoredCandidate], + *, + allowed_ids: set[str], + require_all_ids: bool, +) -> bool: + candidate_ids = [candidate.canonical_field_id for candidate in candidates] + if len(candidate_ids) != len(set(candidate_ids)): + return False + if not set(candidate_ids).issubset(allowed_ids): + return False + if require_all_ids and set(candidate_ids) != allowed_ids: + return False + if [candidate.rank for candidate in candidates] != list(range(1, len(candidates) + 1)): + return False + return all( + left.score >= right.score for left, right in zip(candidates, candidates[1:], strict=False) + ) + + +def _entity_hint(context: DocumentFieldContext) -> str | None: + structural_text = " ".join( + ( + context.section, + *context.row_labels, + *context.nearby_labels, + ) + ) + normalized = normalize_text(structural_text) + matches = { + entity + for entity, terms in _ENTITY_TERMS.items() + if any(normalize_text(term) in normalized for term in terms) + } + return next(iter(matches)) if len(matches) == 1 else None diff --git a/app/documents/dynamic_automation/models.py b/app/documents/dynamic_automation/models.py new file mode 100644 index 0000000..b3a033e --- /dev/null +++ b/app/documents/dynamic_automation/models.py @@ -0,0 +1,122 @@ +"""Strict, value-free contracts for dynamic field mapping.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +RegistryFieldType = Literal[ + "checkbox", + "checkbox_group", + "text", + "date", + "phone", + "number", + "amount", + "signature", + "placeholder", +] + + +class MappingStatus(StrEnum): + MATCHED = "MATCHED" + AMBIGUOUS = "AMBIGUOUS" + UNMAPPED = "UNMAPPED" + NON_DATA = "NON_DATA" + + +class CanonicalSource(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + view: str = Field(pattern=r"^[a-z][a-z0-9_]*$") + column: str = Field(pattern=r"^[a-z][a-z0-9_]*$") + scope_keys: tuple[Literal["tenant_id", "worker_id", "company_id", "task_id"], ...] + + +class CanonicalFieldDefinition(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + field_id: str = Field(pattern=r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$") + entity: str + value_type: str + aliases: tuple[str, ...] + description: str + compatible_field_types: tuple[RegistryFieldType, ...] + repeatable: bool = False + source: CanonicalSource + sensitivity: Literal["public", "business", "personal", "sensitive"] + formatter: str + + +class DocumentFieldContext(BaseModel): + """Bounded structural context for one untrusted document field.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + field_id: str = Field(min_length=1, max_length=200) + container_id: str = Field(default="", max_length=200) + label: str = Field(max_length=200) + normalized_label: str = Field(max_length=200) + field_type: str = Field(min_length=1, max_length=100) + document_title: str = Field(max_length=200) + section: str = Field(max_length=200) + row_labels: tuple[Annotated[str, Field(max_length=200)], ...] = Field(max_length=3) + nearby_labels: tuple[Annotated[str, Field(max_length=200)], ...] = Field(max_length=4) + options: tuple[Annotated[str, Field(max_length=200)], ...] = Field(max_length=50) + repeat_index: int = Field(ge=0) + required: bool + kind: str = Field(min_length=1, max_length=100) + + +class ScoredCandidate(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + canonical_field_id: str = Field(pattern=r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$") + score: float = Field(ge=0, le=1) + rank: int = Field(ge=1) + + +class MappingEvidence(BaseModel): + """Decision evidence without document values or database values.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + reason: str + rule: str | None = None + embedding_rank: int | None = Field(default=None, ge=1) + reranker_score: float | None = Field(default=None, ge=0, le=1) + score_margin: float | None = Field(default=None, ge=0, le=1) + type_compatible: bool | None = None + entity_hint: str | None = None + catalog_version: str + model_version: str | None = None + + +class FieldMapping(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + field_id: str = Field(min_length=1, max_length=200) + repeat_index: int = Field(ge=0) + status: MappingStatus + canonical_field_id: str | None = Field( + default=None, pattern=r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$" + ) + candidates: tuple[ScoredCandidate, ...] = () + evidence: MappingEvidence + + @model_validator(mode="after") + def _matched_mapping_has_canonical_field(self) -> FieldMapping: + if self.status is MappingStatus.MATCHED and self.canonical_field_id is None: + raise ValueError("matched mappings require a canonical_field_id") + if self.status is not MappingStatus.MATCHED and self.canonical_field_id is not None: + raise ValueError("only matched mappings may include a canonical_field_id") + return self + + +class CanonicalMappingPlan(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + catalog_version: str = Field(pattern=r"^v[1-9][0-9]*$") + mappings: tuple[FieldMapping, ...] diff --git a/app/documents/dynamic_automation/ports.py b/app/documents/dynamic_automation/ports.py new file mode 100644 index 0000000..a1bd69f --- /dev/null +++ b/app/documents/dynamic_automation/ports.py @@ -0,0 +1,31 @@ +"""Model-independent ports for canonical field retrieval and reranking.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Protocol + +from .models import CanonicalFieldDefinition, DocumentFieldContext, ScoredCandidate + + +class CandidateRetriever(Protocol): + @property + def model_version(self) -> str: ... + + def retrieve( + self, + context: DocumentFieldContext, + candidates: Sequence[CanonicalFieldDefinition], + top_k: int, + ) -> tuple[ScoredCandidate, ...]: ... + + +class CandidateReranker(Protocol): + @property + def model_version(self) -> str: ... + + def rerank( + self, + context: DocumentFieldContext, + candidates: Sequence[ScoredCandidate], + ) -> tuple[ScoredCandidate, ...]: ... diff --git a/app/documents/dynamic_automation/qwen.py b/app/documents/dynamic_automation/qwen.py new file mode 100644 index 0000000..ae17e47 --- /dev/null +++ b/app/documents/dynamic_automation/qwen.py @@ -0,0 +1,420 @@ +"""Lazy, local-only Qwen3 adapters for canonical field mapping.""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Any, Protocol + +from .models import CanonicalFieldDefinition, DocumentFieldContext, ScoredCandidate + +QWEN3_EMBEDDING_REPO = "Qwen/Qwen3-Embedding-0.6B" +QWEN3_EMBEDDING_REVISION = "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3" +QWEN3_RERANKER_REPO = "Qwen/Qwen3-Reranker-0.6B" +QWEN3_RERANKER_REVISION = "e61197ed45024b0ed8a2d74b80b4d909f1255473" + +QWEN3_EMBEDDING_CACHE_NAME = "qwen3-embedding-0.6b" +QWEN3_RERANKER_CACHE_NAME = "qwen3-reranker-0.6b" + +_RERANKER_SYSTEM_PREFIX = ( + '<|im_start|>system\nJudge whether the Document meets the requirements based on ' + 'the Query and the Instruct provided. Note that the answer can only be "yes" or ' + '"no".<|im_end|>\n<|im_start|>user\n' +) +_RERANKER_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" +_MAPPING_INSTRUCTION = ( + "주어진 문서 필드의 구조적 문맥에 가장 적합한 canonical field를 찾으세요." +) + + +class EmbeddingBackend(Protocol): + def encode_queries( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: ... + + def encode_documents( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: ... + + +class RerankerBackend(Protocol): + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: ... + + +class LocalSentenceTransformerBackend: + """Load SentenceTransformer only on first use and never access the network.""" + + def __init__(self, model_path: str | Path) -> None: + self.model_path = Path(model_path) + self._model: Any = None + + def _get_model(self) -> Any: + if self._model is None: + try: + from sentence_transformers import SentenceTransformer + except ImportError as err: + raise RuntimeError("sentence-transformers is not installed") from err + self._model = SentenceTransformer( + str(self.model_path), + local_files_only=True, + ) + return self._model + + def encode_queries( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + return self._encode(texts, max_length=max_length, batch_size=batch_size) + + def encode_documents( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + return self._encode(texts, max_length=max_length, batch_size=batch_size) + + def _encode( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + model = self._get_model() + model.max_seq_length = max_length + vectors = model.encode( + list(texts), + batch_size=batch_size, + normalize_embeddings=True, + convert_to_numpy=True, + show_progress_bar=False, + ) + return tuple(tuple(float(value) for value in vector) for vector in vectors) + + +class LocalQwen3RerankerBackend: + """Lazy local Transformers backend using Qwen's official yes/no scoring.""" + + def __init__(self, model_path: str | Path) -> None: + self.model_path = Path(model_path) + self._tokenizer: Any = None + self._model: Any = None + + def _get_components(self) -> tuple[Any, Any]: + if self._tokenizer is None or self._model is None: + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as err: + raise RuntimeError("transformers is not installed") from err + tokenizer = AutoTokenizer.from_pretrained( + str(self.model_path), + local_files_only=True, + ) + tokenizer.padding_side = "left" + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForCausalLM.from_pretrained( + str(self.model_path), + local_files_only=True, + ).eval() + self._tokenizer = tokenizer + self._model = model + return self._tokenizer, self._model + + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: + if not pairs: + return () + try: + import torch + except ImportError as err: + raise RuntimeError("torch is not installed") from err + tokenizer, model = self._get_components() + yes_token_id = int(tokenizer.convert_tokens_to_ids("yes")) + no_token_id = int(tokenizer.convert_tokens_to_ids("no")) + prefix_tokens = tokenizer.encode( + _RERANKER_SYSTEM_PREFIX, + add_special_tokens=False, + ) + suffix_tokens = tokenizer.encode( + _RERANKER_SUFFIX, + add_special_tokens=False, + ) + content_max_length = max_length - len(prefix_tokens) - len(suffix_tokens) + if content_max_length < 1: + raise ValueError("max_length is too small for the Qwen3 reranker prompt") + scores: list[float] = [] + for offset in range(0, len(pairs), batch_size): + batch = pairs[offset : offset + batch_size] + contents = [_format_reranker_content(query, document) for query, document in batch] + encoded = tokenizer( + contents, + add_special_tokens=False, + padding=False, + truncation=True, + max_length=content_max_length, + return_attention_mask=False, + ) + input_ids = [ + [*prefix_tokens, *content_tokens, *suffix_tokens] + for content_tokens in encoded["input_ids"] + ] + inputs = tokenizer.pad( + {"input_ids": input_ids}, + padding=True, + max_length=max_length, + return_tensors="pt", + ) + with torch.inference_mode(): + output = model(**inputs) + logits = output.logits[:, -1, :].detach().cpu().tolist() + scores.extend( + _yes_probability(row[yes_token_id], row[no_token_id]) for row in logits + ) + return tuple(scores) + + +class Qwen3EmbeddingRetriever: + def __init__( + self, + model_path: str | Path | None = None, + *, + backend: EmbeddingBackend | None = None, + max_length: int = 512, + batch_size: int = 8, + ) -> None: + if max_length < 1 or max_length > 512: + raise ValueError("max_length must be between 1 and 512") + if batch_size < 1: + raise ValueError("batch_size must be positive") + if backend is None: + local_path = _require_pinned_model_path( + model_path, + cache_name=QWEN3_EMBEDDING_CACHE_NAME, + revision=QWEN3_EMBEDDING_REVISION, + ) + backend = LocalSentenceTransformerBackend(local_path) + self.backend = backend + self.max_length = max_length + self.batch_size = batch_size + + @property + def model_version(self) -> str: + return f"{QWEN3_EMBEDDING_REPO}@{QWEN3_EMBEDDING_REVISION}" + + def retrieve( + self, + context: DocumentFieldContext, + candidates: Sequence[CanonicalFieldDefinition], + top_k: int, + ) -> tuple[ScoredCandidate, ...]: + if top_k < 1: + raise ValueError("top_k must be positive") + if not candidates: + return () + query_vectors = self.backend.encode_queries( + (_format_context(context),), + max_length=self.max_length, + batch_size=self.batch_size, + ) + document_vectors = self.backend.encode_documents( + tuple(_format_definition(candidate) for candidate in candidates), + max_length=self.max_length, + batch_size=self.batch_size, + ) + if len(query_vectors) != 1 or len(document_vectors) != len(candidates): + raise RuntimeError("invalid Qwen3 embedding batch size") + query_vector = query_vectors[0] + scored = [ + ( + _normalized_cosine(query_vector, vector), + candidate.field_id, + ) + for candidate, vector in zip(candidates, document_vectors, strict=True) + ] + scored.sort(key=lambda item: (-item[0], item[1])) + return tuple( + ScoredCandidate(canonical_field_id=field_id, score=score, rank=rank) + for rank, (score, field_id) in enumerate(scored[:top_k], start=1) + ) + + +class Qwen3CandidateReranker: + def __init__( + self, + model_path: str | Path | None = None, + *, + backend: RerankerBackend | None = None, + definition_resolver: Callable[[str], CanonicalFieldDefinition] | None = None, + max_length: int = 512, + batch_size: int = 8, + ) -> None: + if max_length < 1 or max_length > 512: + raise ValueError("max_length must be between 1 and 512") + if batch_size < 1: + raise ValueError("batch_size must be positive") + if backend is None: + local_path = _require_pinned_model_path( + model_path, + cache_name=QWEN3_RERANKER_CACHE_NAME, + revision=QWEN3_RERANKER_REVISION, + ) + backend = LocalQwen3RerankerBackend(local_path) + self.backend = backend + self.definition_resolver = definition_resolver + self.max_length = max_length + self.batch_size = batch_size + + @property + def model_version(self) -> str: + return f"{QWEN3_RERANKER_REPO}@{QWEN3_RERANKER_REVISION}" + + def rerank( + self, + context: DocumentFieldContext, + candidates: Sequence[ScoredCandidate], + ) -> tuple[ScoredCandidate, ...]: + if not candidates: + return () + if self.definition_resolver is None: + raise ValueError("canonical definition_resolver is required for reranking") + definitions: list[CanonicalFieldDefinition] = [] + for candidate in candidates: + try: + definition = self.definition_resolver(candidate.canonical_field_id) + except KeyError as error: + raise ValueError( + f"unknown canonical candidate: {candidate.canonical_field_id}" + ) from error + if definition.field_id != candidate.canonical_field_id: + raise ValueError( + f"definition resolver returned the wrong canonical candidate: " + f"{candidate.canonical_field_id}" + ) + definitions.append(definition) + query = _format_context(context) + scores = self.backend.score_pairs( + tuple((query, _format_definition(definition)) for definition in definitions), + max_length=self.max_length, + batch_size=self.batch_size, + ) + if len(scores) != len(candidates): + raise RuntimeError("invalid Qwen3 reranker batch size") + reranked: list[tuple[float, int, str]] = [] + for candidate, raw_score in zip(candidates, scores, strict=True): + score = float(raw_score) + if not math.isfinite(score) or not 0 <= score <= 1: + raise RuntimeError("invalid Qwen3 reranker probability") + reranked.append((score, candidate.rank, candidate.canonical_field_id)) + reranked.sort(key=lambda item: (-item[0], item[1], item[2])) + return tuple( + ScoredCandidate(canonical_field_id=field_id, score=score, rank=rank) + for rank, (score, _, field_id) in enumerate(reranked, start=1) + ) + + +def _format_context(context: DocumentFieldContext) -> str: + return "\n".join( + ( + f"{context.label} canonical field를 찾으세요.", + f"문서 제목: {context.document_title}", + f"섹션: {context.section}", + f"행 라벨: {', '.join(context.row_labels)}", + f"주변 라벨: {', '.join(context.nearby_labels)}", + f"필드 유형: {context.field_type}", + f"요소 종류: {context.kind}", + ) + ) + + +def _format_definition(candidate: CanonicalFieldDefinition) -> str: + return "\n".join( + ( + f"canonical field: {candidate.field_id}", + f"entity: {candidate.entity}", + f"value type: {candidate.value_type}", + f"aliases: {', '.join(candidate.aliases)}", + f"description: {candidate.description}", + ) + ) + + +def _format_reranker_content(query: str, document: str) -> str: + return ( + f": {_MAPPING_INSTRUCTION}\n\n" + f": {query}\n\n" + f": {document}" + ) + + +def _normalized_cosine(left: Sequence[float], right: Sequence[float]) -> float: + if len(left) != len(right): + raise RuntimeError("Qwen3 embedding dimensions do not match") + left_norm = math.sqrt(sum(value * value for value in left)) + right_norm = math.sqrt(sum(value * value for value in right)) + if left_norm == 0 or right_norm == 0: + return 0.5 + cosine = sum(a * b for a, b in zip(left, right, strict=True)) / ( + left_norm * right_norm + ) + return min(1.0, max(0.0, (cosine + 1.0) / 2.0)) + + +def _require_pinned_model_path( + model_path: str | Path | None, + *, + cache_name: str, + revision: str, +) -> Path: + if model_path is None: + raise ValueError("pinned model cache path is required for a real Qwen3 backend") + path = Path(model_path) + if not path.is_absolute(): + raise ValueError("Qwen3 model_path must be an absolute local path") + if tuple(path.parts[-2:]) != (cache_name, revision): + raise ValueError("Qwen3 model_path must identify the pinned revision directory") + return path.resolve(strict=False) + + +def _yes_probability(yes_logit: float, no_logit: float) -> float: + maximum = max(yes_logit, no_logit) + yes = math.exp(yes_logit - maximum) + no = math.exp(no_logit - maximum) + return yes / (yes + no) + + +__all__ = [ + "LocalQwen3RerankerBackend", + "LocalSentenceTransformerBackend", + "QWEN3_EMBEDDING_REPO", + "QWEN3_EMBEDDING_REVISION", + "QWEN3_RERANKER_REPO", + "QWEN3_RERANKER_REVISION", + "Qwen3CandidateReranker", + "Qwen3EmbeddingRetriever", +] diff --git a/app/documents/dynamic_automation/resources/canonical_fields.v1.yaml b/app/documents/dynamic_automation/resources/canonical_fields.v1.yaml new file mode 100644 index 0000000..613c42d --- /dev/null +++ b/app/documents/dynamic_automation/resources/canonical_fields.v1.yaml @@ -0,0 +1,182 @@ +version: v1 +fields: + - field_id: worker.legal_name + entity: worker + value_type: string + aliases: [근로자 성명, 신청인 성명, Name of employee] + description: Worker's legal name. + compatible_field_types: [text] + source: {view: document_worker_view, column: legal_name, scope_keys: [tenant_id, worker_id]} + sensitivity: personal + formatter: person_name + - field_id: worker.nationality + entity: worker + value_type: string + aliases: [국적, Nationality] + description: Worker's nationality. + compatible_field_types: [text] + source: {view: document_worker_view, column: nationality, scope_keys: [tenant_id, worker_id]} + sensitivity: personal + formatter: string + - field_id: worker.date_of_birth + entity: worker + value_type: date + aliases: [생년월일, Date of birth] + description: Worker's date of birth. + compatible_field_types: [date, text] + source: {view: document_worker_view, column: date_of_birth, scope_keys: [tenant_id, worker_id]} + sensitivity: sensitive + formatter: iso_date + - field_id: worker.phone + entity: worker + value_type: phone + aliases: [근로자 전화번호, Phone number] + description: Worker's phone number. + compatible_field_types: [phone, text] + source: {view: document_worker_view, column: phone, scope_keys: [tenant_id, worker_id]} + sensitivity: personal + formatter: phone + - field_id: worker.email + entity: worker + value_type: email + aliases: [이메일, Email] + description: Worker's email address. + compatible_field_types: [text] + source: {view: document_worker_view, column: email, scope_keys: [tenant_id, worker_id]} + sensitivity: personal + formatter: email + - field_id: company.name + entity: company + value_type: string + aliases: [회사명, 사업장명, Company name] + description: Company's legal name. + compatible_field_types: [text] + source: {view: document_company_view, column: name, scope_keys: [tenant_id, company_id]} + sensitivity: business + formatter: string + - field_id: company.phone + entity: company + value_type: phone + aliases: [사업장 전화번호, 전화번호, Company phone] + description: Company's contact telephone number. + compatible_field_types: [phone, text] + source: {view: document_company_view, column: phone, scope_keys: [tenant_id, company_id]} + sensitivity: business + formatter: phone + - field_id: company.address + entity: company + value_type: address + aliases: [사업장 주소, Address] + description: Company's address. + compatible_field_types: [text] + source: {view: document_company_view, column: address, scope_keys: [tenant_id, company_id]} + sensitivity: business + formatter: address + - field_id: company.representative_name + entity: company + value_type: string + aliases: [대표자 성명, Representative name] + description: Company's representative name. + compatible_field_types: [text] + source: {view: document_company_view, column: representative_name, scope_keys: [tenant_id, company_id]} + sensitivity: personal + formatter: person_name + - field_id: company.business_number + entity: company + value_type: business_number + aliases: [사업자등록번호, Business Registration No.] + description: Company's business registration number. + compatible_field_types: [number, text] + source: {view: document_company_view, column: business_number, scope_keys: [tenant_id, company_id]} + sensitivity: business + formatter: business_number + - field_id: identity.passport_number + entity: identity + value_type: passport_number + aliases: [여권번호, Passport number] + description: Worker's passport number. + compatible_field_types: [text] + source: {view: document_identity_view, column: passport_number, scope_keys: [tenant_id, worker_id]} + sensitivity: sensitive + formatter: string + - field_id: identity.alien_registration_number + entity: identity + value_type: alien_registration_number + aliases: [외국인등록번호, Alien registration number] + description: Worker's alien registration number. + compatible_field_types: [number, text] + source: {view: document_identity_view, column: alien_registration_number, scope_keys: [tenant_id, worker_id]} + sensitivity: sensitive + formatter: string + - field_id: contract.start_date + entity: contract + value_type: date + aliases: [계약 시작일, Contract start date] + description: Contract start date. + compatible_field_types: [date, text] + source: {view: document_task_view, column: contract_start_date, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: iso_date + - field_id: contract.end_date + entity: contract + value_type: date + aliases: [계약 종료일, Contract end date] + description: Contract end date. + compatible_field_types: [date, text] + source: {view: document_task_view, column: contract_end_date, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: iso_date + - field_id: contract.wage + entity: contract + value_type: amount + aliases: [임금, Wage] + description: Contract wage. + compatible_field_types: [amount, number, text] + source: {view: document_task_view, column: wage, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: amount + - field_id: contract.working_hours + entity: contract + value_type: string + aliases: [근로시간, Working hours] + description: Contract working hours. + compatible_field_types: [text] + source: {view: document_task_view, column: working_hours, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: string + - field_id: contract.job_description + entity: contract + value_type: string + aliases: [업무내용, Job description] + description: Contract job description. + compatible_field_types: [text] + source: {view: document_task_view, column: job_description, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: string + - field_id: contract.work_location + entity: contract + value_type: address + aliases: [근무장소, Work location] + description: Contract work location. + compatible_field_types: [text] + source: {view: document_task_view, column: work_location, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: address + - field_id: contract.lodging + entity: contract + value_type: string + aliases: [숙식, Lodging] + description: Contract lodging arrangement. + compatible_field_types: [text] + source: {view: document_task_view, column: lodging, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: string + - field_id: application.date + entity: application + value_type: date + aliases: [신청일, Application date] + description: Application submission date. + compatible_field_types: [date, text] + source: {view: document_task_view, column: application_date, scope_keys: [tenant_id, task_id]} + sensitivity: business + formatter: iso_date diff --git a/app/documents/dynamic_automation/rules.py b/app/documents/dynamic_automation/rules.py new file mode 100644 index 0000000..bcf1d31 --- /dev/null +++ b/app/documents/dynamic_automation/rules.py @@ -0,0 +1,49 @@ +"""Deterministic rules that precede any mapping decision.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .catalog import CanonicalCatalog +from .field_context import normalize_text +from .models import CanonicalFieldDefinition, DocumentFieldContext + +_PROCESS_FLOW_LABELS = frozenset( + map(normalize_text, ("접수", "확인검토", "전산입력", "신청서작성", "고용센터")) +) +_OFFICIAL_USE_LABELS = frozenset( + map(normalize_text, ("관공서용", "공용란", "For Official Use", "결재", "관할관서")) +) +_PAGE_ARROWS = frozenset({"←", "→", "↑", "↓", "◀", "▶", "‹", "›", "«", "»", "<", ">"}) +_NON_DATA_KINDS = frozenset({"official_region", "signable_region"}) + + +@dataclass(frozen=True) +class NonDataDecision: + is_non_data: bool + reason: str | None = None + + +def classify_non_data(context: DocumentFieldContext) -> NonDataDecision: + """Identify regions and labels that can never receive document data.""" + normalized_label = normalize_text(context.label) + if context.kind in _NON_DATA_KINDS: + return NonDataDecision(is_non_data=True, reason=context.kind) + if context.label.strip() in _PAGE_ARROWS: + return NonDataDecision(is_non_data=True, reason="page_navigation_label") + if normalized_label in _PROCESS_FLOW_LABELS: + return NonDataDecision(is_non_data=True, reason="process_flow_label") + if normalized_label in _OFFICIAL_USE_LABELS: + return NonDataDecision(is_non_data=True, reason="official_use_label") + return NonDataDecision(is_non_data=False) + + +def exact_alias_matches( + context: DocumentFieldContext, catalog: CanonicalCatalog +) -> tuple[CanonicalFieldDefinition, ...]: + """Return compatible canonical candidates whose aliases exactly match the label.""" + return tuple( + field + for field in catalog.compatible(context) + if any(normalize_text(alias) == normalize_text(context.label) for alias in field.aliases) + ) diff --git a/app/documents/dynamic_automation/training.py b/app/documents/dynamic_automation/training.py new file mode 100644 index 0000000..def9b2c --- /dev/null +++ b/app/documents/dynamic_automation/training.py @@ -0,0 +1,513 @@ +"""Privacy-safe training datasets and fail-closed model promotion gates.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from .catalog import CanonicalCatalog +from .feedback import MappingFeedbackRecord, ReviewerDecision +from .qwen import ( + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_REPO, + QWEN3_RERANKER_REVISION, +) + +AUTO_PRECISION_FLOOR = 0.99 +SENSITIVE_PRECISION_FLOOR = 0.995 +TRAINING_CODE_VERSION = "dynamic-mapping-training-v2" +EVALUATION_CODE_VERSION = "dynamic-mapping-evaluation-v2" + +_CANONICAL_ID_PATTERN = r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$" +_SHA256_PATTERN = r"^[0-9a-f]{64}$" +_BoundedCanonicalId = Annotated[ + str, Field(max_length=200, pattern=_CANONICAL_ID_PATTERN) +] + + +class TrainingExample(BaseModel): + """One reviewer-approved, value-free retrieval query and label.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + document_layout_hash: str = Field(pattern=_SHA256_PATTERN) + document_kind: str = Field(min_length=1, max_length=100) + document_version: str = Field(min_length=1, max_length=100) + source_institution: str = Field(min_length=1, max_length=100) + field_context_hash: str = Field(pattern=_SHA256_PATTERN) + field_id: str = Field(min_length=1, max_length=200) + repeat_index: int = Field(ge=0) + query_text: str = Field(min_length=1, max_length=1800) + canonical_field_id: _BoundedCanonicalId + catalog_version: str = Field(max_length=20, pattern=r"^v[1-9][0-9]*$") + + +class TrainingSplit(BaseModel): + """Deterministic split whose layout groups never cross partitions.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + train: tuple[TrainingExample, ...] + test: tuple[TrainingExample, ...] + + +class TrainingPair(BaseModel): + """Type-compatible negative pair for retrieval or pairwise ranking.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + document_layout_hash: str = Field(pattern=_SHA256_PATTERN) + query_text: str = Field(min_length=1, max_length=1800) + positive_canonical_field_id: _BoundedCanonicalId + negative_canonical_field_id: _BoundedCanonicalId + + +class EvaluationMetricsEvidence(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + auto_precision: float = Field(ge=0, le=1) + sensitive_precision: float = Field(ge=0, le=1) + coverage: float = Field(ge=0, le=1) + expected_calibration_error: float = Field(ge=0, le=1) + p95_latency_ms: float = Field(ge=0) + + +class UnseenFieldEvidence(BaseModel): + """Generated catalog-field retrieval evidence, never a self-asserted boolean.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + case_id: str = Field(min_length=1, max_length=200) + canonical_field_id: _BoundedCanonicalId + query_sha256: str = Field(pattern=_SHA256_PATTERN) + candidate_ids: tuple[_BoundedCanonicalId, ...] = Field(max_length=20) + retrieved_rank: int | None = Field(default=None, ge=1, le=20) + + +class HeldOutEvaluationReport(BaseModel): + """Hashed evidence produced by loading and executing an exported artifact.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["dynamic-mapping-held-out-v2"] + evaluation_code_version: Literal["dynamic-mapping-evaluation-v2"] + model_artifact_sha256: str = Field(pattern=_SHA256_PATTERN) + dataset_sha256: str = Field(pattern=_SHA256_PATTERN) + catalog_sha256: str = Field(pattern=_SHA256_PATTERN) + catalog_version: str = Field(max_length=20, pattern=r"^v[1-9][0-9]*$") + sample_count: int = Field(ge=1) + cohort_count: int = Field(ge=1) + model_execution_count: int = Field(ge=1) + metrics: EvaluationMetricsEvidence + unseen_field_evidence: UnseenFieldEvidence + + +class ModelManifest(BaseModel): + """Immutable evidence used to compare a candidate with the pinned baseline.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["dynamic-mapping-model-manifest-v2"] + model_kind: Literal[ + "qwen_baseline", "domain_bi_encoder", "domain_pair_reranker" + ] + base_model_repo: str = Field(min_length=1, max_length=300) + base_model_revision: str = Field(min_length=1, max_length=200) + dataset_sha256: str = Field(pattern=_SHA256_PATTERN) + catalog_sha256: str = Field(pattern=_SHA256_PATTERN) + model_artifact_sha256: str = Field(pattern=_SHA256_PATTERN) + evaluation_report_sha256: str = Field(pattern=_SHA256_PATTERN) + catalog_version: str = Field(max_length=20, pattern=r"^v[1-9][0-9]*$") + training_code_version: Literal["dynamic-mapping-training-v2"] + evaluation_code_version: Literal["dynamic-mapping-evaluation-v2"] + training_sample_count: int = Field(ge=1) + evaluation_sample_count: int = Field(ge=1) + training_cohort_count: int = Field(ge=1) + evaluation_cohort_count: int = Field(ge=1) + auto_precision: float = Field(ge=0, le=1) + sensitive_precision: float = Field(ge=0, le=1) + coverage: float = Field(ge=0, le=1) + expected_calibration_error: float = Field(ge=0, le=1) + p95_latency_ms: float = Field(ge=0) + seed: int = Field(default=42, ge=0) + training_canonical_field_ids: tuple[_BoundedCanonicalId, ...] = Field( + default=(), max_length=10_000 + ) + catalog_field_ids: tuple[_BoundedCanonicalId, ...] = Field( + min_length=1, max_length=10_000 + ) + + @field_validator( + "dataset_sha256", + "catalog_sha256", + "model_artifact_sha256", + "evaluation_report_sha256", + ) + @classmethod + def _hash_is_not_a_placeholder(cls, value: str) -> str: + if value == "0" * 64: + raise ValueError("evidence SHA-256 must not be the zero placeholder") + return value + + @field_validator("catalog_field_ids") + @classmethod + def _catalog_membership_is_unique_and_stable( + cls, value: tuple[str, ...] + ) -> tuple[str, ...]: + if len(set(value)) != len(value): + raise ValueError("catalog_field_ids must not contain duplicates") + return tuple(sorted(value)) + + +class PromotionDecision(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + promote: bool + reasons: tuple[str, ...] + + +def build_training_split( + records: Sequence[MappingFeedbackRecord | Mapping[str, Any]], +) -> TrainingSplit: + """Build a stable split after revalidating every sanitized feedback record.""" + validated = tuple(_validate_feedback(record) for record in records) + examples = tuple( + sorted( + ( + _to_example(record) + for record in validated + if record.final_canonical_field_id is not None + and record.decision + in (ReviewerDecision.ACCEPTED, ReviewerDecision.CORRECTED) + ), + key=_example_sort_key, + ) + ) + components = _group_components(examples) + if len(components) < 2: + test_indices: set[int] = set() + else: + test_count = max(1, len(components) // 5) + ranked_components = sorted( + components, + key=lambda component: ( + hashlib.sha256( + "\n".join( + _example_group_fingerprint(examples[index]) + for index in component + ).encode("utf-8") + ).hexdigest(), + tuple(_example_sort_key(examples[index]) for index in component), + ), + ) + test_indices = { + index + for component in ranked_components[:test_count] + for index in component + } + return TrainingSplit( + train=tuple( + example + for index, example in enumerate(examples) + if index not in test_indices + ), + test=tuple( + example + for index, example in enumerate(examples) + if index in test_indices + ), + ) + + +def build_hard_negatives( + split: TrainingSplit, catalog: CanonicalCatalog +) -> tuple[TrainingPair, ...]: + """Return deterministic catalog negatives, with known entity confusions first.""" + definitions = catalog.definitions + pairs: list[TrainingPair] = [] + for example in split.train: + positive = catalog.get(example.canonical_field_id) + compatible = [ + candidate + for candidate in definitions + if candidate.field_id != positive.field_id + and set(candidate.compatible_field_types) + & set(positive.compatible_field_types) + ] + priority = _CONFUSION_PRIORITY.get(positive.field_id, ()) + compatible.sort( + key=lambda candidate: ( + priority.index(candidate.field_id) + if candidate.field_id in priority + else len(priority), + candidate.field_id, + ) + ) + pairs.extend( + TrainingPair( + document_layout_hash=example.document_layout_hash, + query_text=example.query_text, + positive_canonical_field_id=positive.field_id, + negative_canonical_field_id=candidate.field_id, + ) + for candidate in compatible + ) + return tuple(pairs) + + +def compare_manifests( + *, + baseline: ModelManifest, + candidate: ModelManifest, + baseline_report: HeldOutEvaluationReport, + candidate_report: HeldOutEvaluationReport, + baseline_artifact_sha256: str | None = None, + candidate_artifact_sha256: str | None = None, + baseline_report_sha256: str | None = None, + candidate_report_sha256: str | None = None, +) -> PromotionDecision: + """Require every safety, quality, efficiency, and generalization gate.""" + reasons: list[str] = [] + expected_base: tuple[str, str] | None + if candidate.model_kind == "domain_bi_encoder": + expected_base = (QWEN3_EMBEDDING_REPO, QWEN3_EMBEDDING_REVISION) + elif candidate.model_kind == "domain_pair_reranker": + expected_base = (QWEN3_RERANKER_REPO, QWEN3_RERANKER_REVISION) + else: + expected_base = None + reasons.append("candidate_model_kind") + if baseline.model_kind != "qwen_baseline": + reasons.append("baseline_model_kind") + if expected_base is not None and ( + (baseline.base_model_repo, baseline.base_model_revision) != expected_base + or (candidate.base_model_repo, candidate.base_model_revision) != expected_base + ): + reasons.append("base_model_manifest") + baseline_evidence_valid = _manifest_matches_evidence( + baseline, + baseline_report, + artifact_sha256=baseline_artifact_sha256, + report_sha256=baseline_report_sha256, + ) + candidate_evidence_valid = _manifest_matches_evidence( + candidate, + candidate_report, + artifact_sha256=candidate_artifact_sha256, + report_sha256=candidate_report_sha256, + ) + if not baseline_evidence_valid: + reasons.append("baseline_evaluation_evidence") + if not candidate_evidence_valid: + reasons.append("candidate_evaluation_evidence") + + baseline_metrics = baseline_report.metrics + candidate_metrics = candidate_report.metrics + if ( + candidate_metrics.auto_precision < AUTO_PRECISION_FLOOR + or candidate_metrics.auto_precision < baseline_metrics.auto_precision + ): + reasons.append("auto_precision") + if ( + candidate_metrics.sensitive_precision < SENSITIVE_PRECISION_FLOOR + or candidate_metrics.sensitive_precision < baseline_metrics.sensitive_precision + ): + reasons.append("sensitive_precision") + if ( + candidate_metrics.expected_calibration_error + > baseline_metrics.expected_calibration_error + ): + reasons.append("expected_calibration_error") + if candidate_metrics.coverage < baseline_metrics.coverage: + reasons.append("coverage") + if candidate_metrics.p95_latency_ms > baseline_metrics.p95_latency_ms: + reasons.append("p95_latency_ms") + if ( + candidate_metrics.coverage == baseline_metrics.coverage + and candidate_metrics.p95_latency_ms == baseline_metrics.p95_latency_ms + ): + reasons.append("coverage_or_p95_latency_ms") + if candidate.catalog_version != baseline.catalog_version: + reasons.append("catalog_version") + if candidate.dataset_sha256 != baseline.dataset_sha256: + reasons.append("dataset_sha256") + if candidate.catalog_sha256 != baseline.catalog_sha256: + reasons.append("catalog_sha256") + if candidate.catalog_field_ids != baseline.catalog_field_ids: + reasons.append("catalog_field_ids") + unseen = candidate_report.unseen_field_evidence + unseen_rank_valid = ( + unseen.retrieved_rank is not None + and unseen.retrieved_rank <= len(unseen.candidate_ids) + and unseen.candidate_ids[unseen.retrieved_rank - 1] == unseen.canonical_field_id + ) + if ( + not unseen_rank_valid + or unseen.canonical_field_id not in candidate.catalog_field_ids + or unseen.canonical_field_id not in baseline.catalog_field_ids + or unseen.canonical_field_id in candidate.training_canonical_field_ids + ): + reasons.append("unseen_catalog_retrieval") + return PromotionDecision(promote=not reasons, reasons=tuple(reasons)) + + +def held_out_evaluation_report_bytes(report: HeldOutEvaluationReport) -> bytes: + serialized = json.dumps( + report.model_dump(mode="json"), + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + return (serialized + "\n").encode("utf-8") + + +def held_out_evaluation_report_sha256(report: HeldOutEvaluationReport) -> str: + return hashlib.sha256(held_out_evaluation_report_bytes(report)).hexdigest() + + +def _manifest_matches_evidence( + manifest: ModelManifest, + report: HeldOutEvaluationReport, + *, + artifact_sha256: str | None, + report_sha256: str | None, +) -> bool: + metrics = report.metrics + return ( + artifact_sha256 == manifest.model_artifact_sha256 + and report_sha256 == manifest.evaluation_report_sha256 + and report.model_artifact_sha256 == manifest.model_artifact_sha256 + and report.dataset_sha256 == manifest.dataset_sha256 + and report.catalog_sha256 == manifest.catalog_sha256 + and report.catalog_version == manifest.catalog_version + and report.evaluation_code_version == manifest.evaluation_code_version + and report.sample_count == manifest.evaluation_sample_count + and report.cohort_count == manifest.evaluation_cohort_count + and metrics.auto_precision == manifest.auto_precision + and metrics.sensitive_precision == manifest.sensitive_precision + and metrics.coverage == manifest.coverage + and metrics.expected_calibration_error + == manifest.expected_calibration_error + and metrics.p95_latency_ms == manifest.p95_latency_ms + ) + + +def training_dataset_sha256(split: TrainingSplit) -> str: + """Hash canonical JSON so record ordering and JSONL whitespace cannot affect identity.""" + payload = split.model_dump(mode="json") + serialized = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(serialized).hexdigest() + + +def _validate_feedback( + record: MappingFeedbackRecord | Mapping[str, Any], +) -> MappingFeedbackRecord: + payload: Any + if isinstance(record, MappingFeedbackRecord): + payload = record.model_dump(mode="json", warnings="none") + else: + payload = record + return MappingFeedbackRecord.model_validate(payload) + + +def _to_example(record: MappingFeedbackRecord) -> TrainingExample: + assert record.final_canonical_field_id is not None + parts = ( + f"label: {record.label}", + f"section: {record.section}", + f"row labels: {' | '.join(record.row_labels)}", + f"nearby labels: {' | '.join(record.nearby_labels)}", + ) + return TrainingExample( + document_layout_hash=record.layout_hash, + document_kind=record.document_kind, + document_version=record.document_version, + source_institution=record.source_institution, + field_context_hash=record.field_context_hash, + field_id=record.field_id, + repeat_index=record.repeat_index, + query_text="\n".join(parts), + canonical_field_id=record.final_canonical_field_id, + catalog_version=record.catalog_version, + ) + + +def _example_sort_key(example: TrainingExample) -> tuple[str, str, str, int]: + return ( + example.document_layout_hash, + example.field_context_hash, + example.field_id, + example.repeat_index, + ) + + +def _group_components(examples: Sequence[TrainingExample]) -> tuple[tuple[int, ...], ...]: + """Return connected components sharing any required group identity.""" + parents = list(range(len(examples))) + + def find(index: int) -> int: + while parents[index] != index: + parents[index] = parents[parents[index]] + index = parents[index] + return index + + def union(left: int, right: int) -> None: + left_root = find(left) + right_root = find(right) + if left_root != right_root: + parents[max(left_root, right_root)] = min(left_root, right_root) + + seen: dict[tuple[str, str], int] = {} + for index, example in enumerate(examples): + identities = ( + ("layout_hash", example.document_layout_hash), + ("document_kind", example.document_kind), + ("document_version", example.document_version), + ("source_institution", example.source_institution), + ) + for identity in identities: + previous = seen.setdefault(identity, index) + union(index, previous) + + components: dict[int, list[int]] = {} + for index in range(len(examples)): + components.setdefault(find(index), []).append(index) + return tuple(tuple(indices) for _, indices in sorted(components.items())) + + +def _example_group_fingerprint(example: TrainingExample) -> str: + return "|".join( + ( + example.document_layout_hash, + example.document_kind, + example.document_version, + example.source_institution, + example.field_context_hash, + example.field_id, + str(example.repeat_index), + ) + ) + + +_CONFUSION_PRIORITY: dict[str, tuple[str, ...]] = { + "worker.phone": ("company.phone", "guarantor.phone"), + "company.phone": ("worker.phone", "guarantor.phone"), + "guarantor.phone": ("worker.phone", "company.phone"), + "worker.legal_name": ("company.representative_name",), + "company.representative_name": ("worker.legal_name",), + "identity.passport_number": ("identity.alien_registration_number",), + "identity.alien_registration_number": ("identity.passport_number",), + "contract.start_date": ("application.date", "contract.end_date", "contract.expiry_date"), + "contract.end_date": ("contract.start_date", "application.date", "contract.expiry_date"), + "contract.expiry_date": ("contract.end_date", "application.date", "contract.start_date"), + "application.date": ("contract.start_date", "contract.end_date", "contract.expiry_date"), +} diff --git a/docs/evaluations/dynamic-document-mapping-baseline.md b/docs/evaluations/dynamic-document-mapping-baseline.md new file mode 100644 index 0000000..6e8588f --- /dev/null +++ b/docs/evaluations/dynamic-document-mapping-baseline.md @@ -0,0 +1,54 @@ +# Dynamic Document Mapping Baseline + +```yaml +date: 2026-08-11 +mode: rule +catalog_version: v1 +fixture_cases: 5 +gate: PASSED +``` + +This deterministic offline baseline uses +`tests/fixtures/dynamic_automation/mapping_cases.jsonl`. Rule mode exercises exact aliases, +fail-closed ambiguity, non-data classification, and unsupported fields without importing or +requiring model packages. The fixture contains labels and bounded structural context only; it +contains no document values or resolved database values. + +## Results + +| Metric | Rule baseline | +| --- | ---: | +| Extraction precision | 1.000000 | +| Extraction recall | 1.000000 | +| Top-1 accuracy | 1.000000 | +| Top-5 recall | 1.000000 | +| Automatic-match precision | 1.000000 | +| Coverage | 0.500000 | +| Ambiguous accuracy | 1.000000 | +| Sensitive-field precision | 1.000000 | +| Document zero-error rate | 1.000000 | + +The release gate requires automatic-match precision of at least `0.99` and sensitive-field +precision of at least `0.995`. This five-case rule baseline passes both gates. Coverage is lower by +design: uncertain fields are deferred instead of being accepted automatically. + +## Metric definitions + +- Extraction precision and recall treat every status other than `NON_DATA` as an extracted data + field. +- Top-1 accuracy and top-k recall compare literal expected canonical IDs with the ranked candidate + IDs. +- Automatic-match precision measures correct `MATCHED` outcomes among automatic matches; coverage + measures automatic matches among expected data fields. +- Ambiguous accuracy measures correct deferral for cases labeled `AMBIGUOUS`. +- Sensitive-field precision covers automatic matches where either the expected field or the + predicted canonical target is catalog-marked `sensitive`, so false assignments into sensitive + targets cannot escape the release gate. +- Document zero-error rate requires every evaluated field in a document to have the expected status + and, for a match, the expected canonical ID. + +Reproduce the JSON report with: + +```powershell +python scripts/evaluate_dynamic_mapping.py --cases tests/fixtures/dynamic_automation/mapping_cases.jsonl --catalog app/documents/dynamic_automation/resources/canonical_fields.v1.yaml --mode rule --output build/dynamic-mapping-baseline.json +``` diff --git a/docs/superpowers/plans/2026-08-12-dynamic-automation-config-isolation.md b/docs/superpowers/plans/2026-08-12-dynamic-automation-config-isolation.md new file mode 100644 index 0000000..da8e813 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-dynamic-automation-config-isolation.md @@ -0,0 +1,303 @@ +# Dynamic Automation Configuration Isolation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove dynamic document-mapping configuration from the FastAPI server's shared settings path while preserving the existing opt-in environment interface for dynamic evaluation commands. + +**Architecture:** A package-owned `DynamicAutomationSettings` model reads and validates the existing environment variables only when the Qwen evaluation path constructs it. `app.core.config.Settings` returns to its prior server-only contract, so malformed dynamic-only environment values cannot affect `app.main.create_app()`. + +**Tech Stack:** Python 3.12, Pydantic v2, pydantic-settings, pytest, Ruff + +## Global Constraints + +- Preserve all existing `FOWOCO_DYNAMIC_AUTOMATION_*` environment variable names and defaults. +- Keep Qwen model paths pinned below `FOWOCO_MODEL_CACHE_DIR`. +- Do not import, load, or download model weights while reading configuration. +- Do not change API routes, schemas, workflow graphs, document editors, MCP behavior, or database access. +- Use strict TDD: observe each new regression test fail before production edits. + +--- + +### Task 1: Isolate dynamic automation settings from server startup + +**Files:** +- Create: `app/documents/dynamic_automation/config.py` +- Modify: `app/core/config.py:1-162` +- Modify: `scripts/evaluate_dynamic_mapping.py:320-356` +- Modify: `tests/documents/dynamic_automation/test_mapping_config.py` +- Modify: `tests/integration/dynamic_automation/test_qwen_mapping_smoke.py` + +**Interfaces:** +- Consumes: `QWEN3_EMBEDDING_CACHE_NAME`, `QWEN3_EMBEDDING_REVISION`, `QWEN3_RERANKER_CACHE_NAME`, and `QWEN3_RERANKER_REVISION` from `app.documents.dynamic_automation.qwen`. +- Produces: `DynamicAutomationSettings(BaseSettings)` with `model_cache_dir: Path`, `dynamic_automation_mapping_enabled: bool`, `dynamic_automation_embedding_model_path: Path | None`, `dynamic_automation_reranker_model_path: Path | None`, `dynamic_automation_min_reranker_score: float`, and `dynamic_automation_min_margin: float`. +- Produces: unchanged Qwen evaluation behavior through `_make_mapper(...)`; rule mode does not construct dynamic settings. + +- [x] **Step 1: Write server-isolation and package-settings tests** + +Update imports and add literal behavioral assertions in `test_mapping_config.py`: + +```python +import os + +from app.core.config import Settings +from app.documents.dynamic_automation.config import DynamicAutomationSettings + + +def test_invalid_dynamic_environment_cannot_break_server_startup() -> None: + environment = { + **os.environ, + "FOWOCO_DYNAMIC_AUTOMATION_MIN_MARGIN": "not-a-number", + "FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH": "outside-cache/model", + } + completed = subprocess.run( + [sys.executable, "-c", "from app.main import create_app; create_app()"], + cwd=Path(__file__).parents[3], + env=environment, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + + +def test_shared_settings_do_not_expose_dynamic_automation_fields() -> None: + settings = Settings(_env_file=None) + assert not hasattr(settings, "dynamic_automation_mapping_enabled") + assert not hasattr(settings, "dynamic_automation_embedding_model_path") +``` + +Change the existing defaults, explicit-environment, managed-cache, pinned +revision, and probability-bound tests to instantiate +`DynamicAutomationSettings` instead of shared `Settings`. + +- [x] **Step 2: Run tests to verify RED** + +Run: + +```text +python -m pytest tests/documents/dynamic_automation/test_mapping_config.py -o addopts='' -q +``` + +Expected: collection fails because `app.documents.dynamic_automation.config` +does not exist. Do not edit production code until this exact missing-module +failure is observed. + +- [x] **Step 3: Add the package-owned settings model** + +Create `app/documents/dynamic_automation/config.py` with this public shape: + +```python +from pathlib import Path +import tempfile +from typing import Self + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +from .qwen import ( + QWEN3_EMBEDDING_CACHE_NAME, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_CACHE_NAME, + QWEN3_RERANKER_REVISION, +) + +_EMBEDDING_PATH = Path(QWEN3_EMBEDDING_CACHE_NAME) / QWEN3_EMBEDDING_REVISION +_RERANKER_PATH = Path(QWEN3_RERANKER_CACHE_NAME) / QWEN3_RERANKER_REVISION + + +class DynamicAutomationSettings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + env_prefix="FOWOCO_", + extra="ignore", + ) + + model_cache_dir: Path = Field( + default_factory=lambda: Path(tempfile.gettempdir()) / "fowoco-model-cache" + ) + dynamic_automation_mapping_enabled: bool = False + dynamic_automation_embedding_model_path: Path | None = None + dynamic_automation_reranker_model_path: Path | None = None + dynamic_automation_min_reranker_score: float = Field(default=0.90, ge=0, le=1) + dynamic_automation_min_margin: float = Field(default=0.10, ge=0, le=1) + + @model_validator(mode="after") + def derive_model_paths(self) -> Self: + embedding_path = self.dynamic_automation_embedding_model_path or ( + self.model_cache_dir / _EMBEDDING_PATH + ) + reranker_path = self.dynamic_automation_reranker_model_path or ( + self.model_cache_dir / _RERANKER_PATH + ) + self.dynamic_automation_embedding_model_path = _managed_model_path( + embedding_path, + model_cache_dir=self.model_cache_dir, + pinned_suffix=_EMBEDDING_PATH, + setting_name="dynamic_automation_embedding_model_path", + ) + self.dynamic_automation_reranker_model_path = _managed_model_path( + reranker_path, + model_cache_dir=self.model_cache_dir, + pinned_suffix=_RERANKER_PATH, + setting_name="dynamic_automation_reranker_model_path", + ) + return self + + +def _managed_model_path( + path: Path, + *, + model_cache_dir: Path, + pinned_suffix: Path, + setting_name: str, +) -> Path: + resolved_cache = model_cache_dir.resolve(strict=False) + resolved_path = path.resolve(strict=False) + try: + relative_path = resolved_path.relative_to(resolved_cache) + except ValueError as err: + raise ValueError(f"{setting_name} must be below model_cache_dir") from err + if relative_path.parts[-2:] != pinned_suffix.parts: + raise ValueError( + f"{setting_name} must end in the pinned revision directory " + f"{pinned_suffix.as_posix()}" + ) + return resolved_path +``` + +This preserves the existing deterministic path derivation and managed-cache +validation while making the Qwen constants the single source of truth. Keep +`_managed_model_path(...)` private to this new module. + +- [x] **Step 4: Remove dynamic behavior from shared settings** + +Delete only the two dynamic pinned-path constants, the five dynamic fields, +`derive_dynamic_automation_model_paths`, and `_managed_model_path` from +`app/core/config.py`. Preserve `model_cache_dir` because existing language model +composition and download behavior use it. Remove `Self` from imports only if no +remaining validator needs it; the OCR validator still returns `Self`, so retain +it. + +- [x] **Step 5: Rewire only the Qwen evaluation path** + +In the non-rule branch of `_make_mapper` replace: + +```python +from app.core.config import Settings +settings = Settings() +``` + +with: + +```python +from app.documents.dynamic_automation.config import DynamicAutomationSettings +settings = DynamicAutomationSettings() +``` + +Do not import this module at script top level. Keeping the import inside the +Qwen branch ensures rule evaluation and FastAPI startup do not construct the +dynamic settings model. + +Update the conditional real-model smoke to construct +`DynamicAutomationSettings` as well. It is a dynamic-only consumer and must no +longer depend on the shared FastAPI settings object. + +- [x] **Step 6: Run focused tests to verify GREEN** + +Run: + +```text +python -m pytest tests/documents/dynamic_automation/test_mapping_config.py tests/documents/dynamic_automation/test_evaluation.py -o addopts='' -q +``` + +Expected: all selected tests pass, including the subprocess server-startup +regression and existing fail-closed Qwen CLI cases. + +- [x] **Step 7: Run scoped regression and quality gates** + +Run: + +```text +python -m pytest tests/documents/dynamic_automation -o addopts='' -q +python -m pytest tests/integration/dynamic_automation/test_qwen_mapping_smoke.py -o addopts='' -q +python -m pytest tests/api/test_analyses_endpoint.py tests/api/test_internal_handshake.py tests/contracts/test_analyses_fixtures.py -o addopts='' -q +python -m ruff check app/core/config.py app/documents/dynamic_automation scripts/evaluate_dynamic_mapping.py tests/documents/dynamic_automation/test_mapping_config.py +git diff --check +``` + +Expected: dynamic automation and Server↔AI contract suites pass; Ruff and diff +checks exit 0. + +- [x] **Step 8: Record the repository-wide baseline** + +Run: + +```text +python -m pytest -q --tb=no +``` + +Expected on the current base: 57 known failures in Language context checksum, +LangGraph compatibility, Qdrant, Compose, and runtime composition. Confirm no +new failure appears outside that recorded set. This result must be disclosed in +the Draft PR; it is not a success gate for making the feature merge-ready. + +- [x] **Step 9: Commit the isolated implementation** + +```text +git add app/core/config.py app/documents/dynamic_automation/config.py scripts/evaluate_dynamic_mapping.py tests/documents/dynamic_automation/test_mapping_config.py tests/integration/dynamic_automation/test_qwen_mapping_smoke.py docs/superpowers/plans/2026-08-12-dynamic-automation-config-isolation.md +git commit -m "fix(doc-automation): isolate mapping configuration" +``` + +The commit must contain only these six files. + +--- + +### Task 2: Publish a Draft pull request with explicit boundaries + +**Files:** +- Inspect: `.github/PULL_REQUEST_TEMPLATE.md` or repository PR templates if present +- No repository file changes are required. + +**Interfaces:** +- Consumes: branch `feat/dynamic-field-mapping-foundation` and its verified commit history. +- Produces: a Draft GitHub pull request against the confirmed fork base `feat/mcp_mapping`. + +- [x] **Step 1: Confirm branch provenance and remote base** + +Run: + +```text +git merge-base --is-ancestor feat/mcp_mapping HEAD +git ls-remote --heads origin feat/mcp_mapping feat/dynamic-field-mapping-foundation +git status --short +``` + +Expected: the local feature contains `feat/mcp_mapping`, the worktree is clean, +and the remote base exists. If the remote base does not exist, stop and ask for +the intended GitHub base rather than targeting `develop` or `main` by guess. + +- [x] **Step 2: Push without force** + +```text +git push -u origin feat/dynamic-field-mapping-foundation +``` + +Expected: push succeeds without rewriting any remote history. + +- [x] **Step 3: Create the Draft PR** + +Use GitHub CLI against `fowoco/ai` with base `feat/mcp_mapping`. The PR body must +state: + +- additive dynamic mapping package; existing template path is unchanged; +- MCP registry is consumed but MCP extraction itself is not replaced; +- no DB access, SQL generation, or Server runtime wiring is included; +- dynamic configuration is isolated from FastAPI shared settings; +- scoped test counts and Server↔AI contract test count; +- the repository-wide 57-failure pre-existing baseline; +- real Qwen smoke was skipped when the pinned cache was unavailable and no + weights were downloaded. + +Create it as Draft, retain the linked worktree for review fixes, and report the +PR URL. diff --git a/docs/superpowers/specs/2026-08-12-dynamic-automation-config-isolation-design.md b/docs/superpowers/specs/2026-08-12-dynamic-automation-config-isolation-design.md new file mode 100644 index 0000000..a4bdb28 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-dynamic-automation-config-isolation-design.md @@ -0,0 +1,84 @@ +# Dynamic Automation Configuration Isolation Design + +## Goal + +Keep the dynamic document-mapping feature opt-in without changing the shared +configuration path used when the FastAPI server starts. Existing +`FOWOCO_DYNAMIC_AUTOMATION_*` environment variable names remain stable, but +only dynamic-automation commands read them. + +## Alternatives considered + +1. **Package-owned settings (selected).** Move all dynamic mapping fields, + pinned-path derivation, and validation into + `app.documents.dynamic_automation.config`. This provides a hard import and + validation boundary while retaining environment-based operations. +2. **Lazy fields on the shared `Settings`.** Skip validation while disabled. + This reduces startup risk but still changes the server configuration model + and leaves accidental coupling possible. +3. **CLI arguments only.** Remove environment variables and require every + command to receive paths and thresholds explicitly. This isolates the + server but makes repeatable deployment and evaluation unnecessarily harder. + +## Architecture + +`app.core.config.Settings` returns to its pre-feature contract: it contains no +dynamic-automation fields, constants, path derivation, or validators. Because +`app.main.create_app()` only constructs this shared object, server startup does +not import, validate, load, or resolve anything from dynamic automation. + +A new `DynamicAutomationSettings` class belongs to the dynamic-automation +package. It reads: + +- `FOWOCO_MODEL_CACHE_DIR` +- `FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED` +- `FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH` +- `FOWOCO_DYNAMIC_AUTOMATION_RERANKER_MODEL_PATH` +- `FOWOCO_DYNAMIC_AUTOMATION_MIN_RERANKER_SCORE` +- `FOWOCO_DYNAMIC_AUTOMATION_MIN_MARGIN` + +The package-owned class retains the existing pinned-revision and managed-cache +validation. It performs no model import, load, download, or network access. + +Only `scripts/evaluate_dynamic_mapping.py` constructs this settings object, +and only for a Qwen-backed evaluation mode. Rule-only evaluation does not need +model paths. The opt-in model downloader remains independently controlled by +`--include-document-automation` and keeps its existing default behavior. + +## Data and error flow + +When the FastAPI application starts, dynamic environment variables are ignored +by shared settings. Invalid dynamic-only paths therefore cannot prevent the +existing server from starting. + +When a dynamic Qwen evaluation starts, the package-owned settings class derives +default pinned paths below `FOWOCO_MODEL_CACHE_DIR` or validates explicitly +provided paths. Invalid, unpinned, or out-of-cache paths fail before any model +backend is imported. A disabled mapping setting continues to fail closed for a +Qwen evaluation request. + +## Compatibility boundaries + +- No API route, request/response schema, workflow graph, document editor, HWP, + HWPX, or server composition code changes. +- Existing dynamic environment variable names and defaults remain unchanged. +- `.env.example` retains the opt-in documentation. +- The dynamic catalog and mapping behavior remain unchanged. +- No database access or Server canonical-slot integration is added. + +## Verification + +Tests will prove: + +1. Shared `Settings` has no dynamic-automation fields. +2. Invalid `FOWOCO_DYNAMIC_AUTOMATION_*` values cannot break shared `Settings` + construction or FastAPI server startup. +3. Package-owned settings preserve the current defaults, environment parsing, + thresholds, pinned revisions, and managed-cache rejection behavior. +4. The evaluation CLI reads package-owned settings and preserves fail-closed + Qwen behavior. +5. Dynamic-automation tests and the existing analyses contract tests pass. + +The implementation is limited to the shared config rollback, the new package +settings module, the evaluation CLI import, focused tests, and any necessary +documentation adjustments. diff --git a/pyproject.toml b/pyproject.toml index c80a743..c79c4d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,12 @@ language-retrieval = [ "FlagEmbedding>=1.3,<2", "huggingface-hub>=0.36,<2", ] +document-automation = [ + "PyYAML>=6.0,<7", + "sentence-transformers>=5,<6", + "transformers>=4.51,<5", + "torch>=2.2,<3", +] [tool.setuptools.packages.find] where = ["."] include = ["app*"] @@ -57,6 +63,7 @@ include = ["app*"] "app.documents.hwp5.templates" = ["*.json", "*.hwp"] "app.documents.hwpx.templates" = ["*.hwpx"] "app.agents.language.resources" = ["*.json", "*.sha256", "prompts/*.md"] +"app.documents.dynamic_automation.resources" = ["*.yaml"] [tool.pytest.ini_options] addopts = "-q" diff --git a/scripts/compare_dynamic_mapping_models.py b/scripts/compare_dynamic_mapping_models.py new file mode 100644 index 0000000..53c5920 --- /dev/null +++ b/scripts/compare_dynamic_mapping_models.py @@ -0,0 +1,82 @@ +"""Compare dynamic mapping model manifests using fail-closed promotion gates.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +from pydantic import ValidationError + +_PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from app.documents.dynamic_automation.training import ( # noqa: E402 + HeldOutEvaluationReport, + ModelManifest, + compare_manifests, +) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--baseline-artifact", required=True, type=Path) + parser.add_argument("--baseline-report", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--candidate-artifact", required=True, type=Path) + parser.add_argument("--candidate-report", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + + try: + baseline = ModelManifest.model_validate_json( + args.baseline.read_text(encoding="utf-8") + ) + candidate = ModelManifest.model_validate_json( + args.candidate.read_text(encoding="utf-8") + ) + baseline_report_bytes = args.baseline_report.read_bytes() + candidate_report_bytes = args.candidate_report.read_bytes() + baseline_report = HeldOutEvaluationReport.model_validate_json( + baseline_report_bytes + ) + candidate_report = HeldOutEvaluationReport.model_validate_json( + candidate_report_bytes + ) + decision = compare_manifests( + baseline=baseline, + candidate=candidate, + baseline_report=baseline_report, + candidate_report=candidate_report, + baseline_artifact_sha256=_file_sha256(args.baseline_artifact), + candidate_artifact_sha256=_file_sha256(args.candidate_artifact), + baseline_report_sha256=hashlib.sha256(baseline_report_bytes).hexdigest(), + candidate_report_sha256=hashlib.sha256(candidate_report_bytes).hexdigest(), + ) + except (OSError, ValidationError, ValueError) as error: + print(f"comparison failed: {error}", file=sys.stderr) + return 1 + + serialized = json.dumps( + decision.model_dump(mode="json"), + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n", encoding="utf-8") + print(serialized) + return 0 if decision.promote else 2 + + +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/download_language_models.py b/scripts/download_language_models.py index 8f6572b..89e9041 100644 --- a/scripts/download_language_models.py +++ b/scripts/download_language_models.py @@ -37,24 +37,54 @@ }, ] +DOCUMENT_AUTOMATION_MODEL_SPECS: list[dict[str, str]] = [ + { + "name": "qwen3-embedding-0.6b", + "repo": "Qwen/Qwen3-Embedding-0.6B", + "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", + }, + { + "name": "qwen3-reranker-0.6b", + "repo": "Qwen/Qwen3-Reranker-0.6B", + "revision": "e61197ed45024b0ed8a2d74b80b4d909f1255473", + }, +] + # 캐시 히트 판단 기준 파일 _SENTINEL_FILE = "config.json" -def verify_model_cache(cache_dir: Path) -> list[str]: +def _selected_model_specs( + *, include_document_automation: bool = False +) -> list[dict[str, str]]: + if include_document_automation: + return [*MODEL_SPECS, *DOCUMENT_AUTOMATION_MODEL_SPECS] + return MODEL_SPECS + + +def verify_model_cache( + cache_dir: Path, *, include_document_automation: bool = False +) -> list[str]: """캐시 디렉터리에서 누락된 모델 이름 목록 반환. 네트워크 접근 없음. 파일 존재 여부만 확인. """ missing: list[str] = [] - for spec in MODEL_SPECS: + for spec in _selected_model_specs( + include_document_automation=include_document_automation + ): sentinel = cache_dir / spec["name"] / spec["revision"] / _SENTINEL_FILE if not sentinel.exists(): missing.append(spec["name"]) return missing -def download_models(cache_dir: Path, force: bool = False) -> None: +def download_models( + cache_dir: Path, + force: bool = False, + *, + include_document_automation: bool = False, +) -> None: """각 모델을 고정 리비전으로 캐시 디렉터리에 다운로드. huggingface_hub 필요. 미설치 시 ImportError 안내 후 종료. @@ -69,7 +99,9 @@ def download_models(cache_dir: Path, force: bool = False) -> None: ) sys.exit(1) - for spec in MODEL_SPECS: + for spec in _selected_model_specs( + include_document_automation=include_document_automation + ): local_dir = cache_dir / spec["name"] / spec["revision"] sentinel = local_dir / _SENTINEL_FILE if sentinel.exists() and not force: @@ -103,9 +135,17 @@ def main() -> None: action="store_true", help="이미 캐시된 모델도 재다운로드", ) + parser.add_argument( + "--include-document-automation", + action="store_true", + help="Qwen3 document automation models also download/verify", + ) args = parser.parse_args() - missing = verify_model_cache(args.cache_dir) + missing = verify_model_cache( + args.cache_dir, + include_document_automation=args.include_document_automation, + ) if args.verify_only: if missing: print(f"[missing] {', '.join(missing)}", file=sys.stderr) @@ -114,7 +154,11 @@ def main() -> None: return if missing or args.force: - download_models(args.cache_dir, force=args.force) + download_models( + args.cache_dir, + force=args.force, + include_document_automation=args.include_document_automation, + ) else: print("[ok] 모든 모델 캐시 확인 완료 (다운로드 불필요)") diff --git a/scripts/evaluate_dynamic_mapping.py b/scripts/evaluate_dynamic_mapping.py new file mode 100644 index 0000000..2d8bda2 --- /dev/null +++ b/scripts/evaluate_dynamic_mapping.py @@ -0,0 +1,450 @@ +"""Deterministic offline evaluation for dynamic canonical field mapping.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Any + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +AUTO_PRECISION_THRESHOLD = 0.99 +SENSITIVE_PRECISION_THRESHOLD = 0.995 +_CANONICAL_ID_PATTERN = r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$" +_BoundedCanonicalId = Annotated[ + str, Field(max_length=200, pattern=_CANONICAL_ID_PATTERN) +] +_BoundedText = Annotated[str, Field(max_length=200)] + + +class EvaluationStatus(StrEnum): + MATCHED = "MATCHED" + AMBIGUOUS = "AMBIGUOUS" + UNMAPPED = "UNMAPPED" + NON_DATA = "NON_DATA" + + +class EvaluationCase(BaseModel): + """Literal expected and observed outcomes for one document field.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + case_id: str = Field(min_length=1, max_length=200) + document_id: str = Field(min_length=1, max_length=200) + expected_status: EvaluationStatus + expected_canonical_field_id: _BoundedCanonicalId | None = None + expected_sensitive: bool + predicted_status: EvaluationStatus + predicted_canonical_field_id: _BoundedCanonicalId | None = None + predicted_sensitive: bool = False + candidate_ids: tuple[_BoundedCanonicalId, ...] = Field(default=(), max_length=20) + + @model_validator(mode="after") + def _matched_prediction_has_id(self) -> EvaluationCase: + has_id = self.predicted_canonical_field_id is not None + if (self.predicted_status is EvaluationStatus.MATCHED) != has_id: + raise ValueError("only matched predictions may include a canonical field ID") + return self + + +class EvaluationMetrics(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + extraction_precision: float + extraction_recall: float + top_1_accuracy: float + top_k_recall: float + auto_precision: float + coverage: float + ambiguous_accuracy: float + sensitive_field_precision: float + document_zero_error_rate: float + + +class _FixtureContext(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + field_id: str = Field(min_length=1, max_length=200) + label: str = Field(max_length=200) + normalized_label: str = Field(max_length=200) + field_type: str = Field(min_length=1, max_length=100) + document_title: str = Field(max_length=200) + section: str = Field(max_length=200) + row_labels: tuple[_BoundedText, ...] = Field(max_length=3) + nearby_labels: tuple[_BoundedText, ...] = Field(max_length=4) + options: tuple[_BoundedText, ...] = Field(max_length=50) + repeat_index: int = Field(ge=0) + required: bool + kind: str = Field(min_length=1, max_length=100) + + +class _FixtureCase(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + case_id: str = Field(min_length=1, max_length=200) + document_id: str = Field(min_length=1, max_length=200) + context: _FixtureContext + expected_status: EvaluationStatus + expected_canonical_field_id: _BoundedCanonicalId | None = None + + +def evaluate_cases( + cases: Sequence[EvaluationCase | Mapping[str, Any]], *, top_k: int = 5 +) -> EvaluationMetrics: + """Compute mapping metrics solely from literal expected and predicted outcomes.""" + if top_k < 1: + raise ValueError("top_k must be positive") + validated = tuple( + case if isinstance(case, EvaluationCase) else EvaluationCase.model_validate(case) + for case in cases + ) + + actual_data = [ + case for case in validated if case.expected_status is not EvaluationStatus.NON_DATA + ] + predicted_data = [ + case for case in validated if case.predicted_status is not EvaluationStatus.NON_DATA + ] + extraction_hits = sum( + case.expected_status is not EvaluationStatus.NON_DATA for case in predicted_data + ) + + ranked = [case for case in validated if case.expected_canonical_field_id is not None] + top_1_hits = sum( + bool(case.candidate_ids) + and case.candidate_ids[0] == case.expected_canonical_field_id + for case in ranked + ) + top_k_hits = sum( + case.expected_canonical_field_id in case.candidate_ids[:top_k] for case in ranked + ) + + automatic = [ + case for case in validated if case.predicted_status is EvaluationStatus.MATCHED + ] + automatic_on_expected_data = [ + case for case in actual_data if case.predicted_status is EvaluationStatus.MATCHED + ] + correct_automatic = sum(_is_correct(case) for case in automatic) + expected_ambiguous = [ + case for case in validated if case.expected_status is EvaluationStatus.AMBIGUOUS + ] + correct_ambiguous = sum( + case.predicted_status is EvaluationStatus.AMBIGUOUS for case in expected_ambiguous + ) + sensitive_automatic = [ + case for case in automatic if case.expected_sensitive or case.predicted_sensitive + ] + correct_sensitive = sum(_is_correct(case) for case in sensitive_automatic) + + documents: dict[str, list[EvaluationCase]] = defaultdict(list) + for case in validated: + documents[case.document_id].append(case) + zero_error_documents = sum( + all(_is_correct(case) for case in document_cases) + for document_cases in documents.values() + ) + + return EvaluationMetrics( + extraction_precision=_ratio(extraction_hits, len(predicted_data)), + extraction_recall=_ratio(extraction_hits, len(actual_data)), + top_1_accuracy=_ratio(top_1_hits, len(ranked)), + top_k_recall=_ratio(top_k_hits, len(ranked)), + auto_precision=_ratio(correct_automatic, len(automatic)), + coverage=_ratio(len(automatic_on_expected_data), len(actual_data)), + ambiguous_accuracy=_ratio(correct_ambiguous, len(expected_ambiguous)), + sensitive_field_precision=_ratio(correct_sensitive, len(sensitive_automatic)), + document_zero_error_rate=_ratio(zero_error_documents, len(documents)), + ) + + +def main( + argv: Sequence[str] | None = None, + *, + embedding_backend: Any | None = None, + reranker_backend: Any | None = None, +) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cases", required=True, type=Path) + parser.add_argument("--catalog", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--mode", required=True, choices=("rule", "qwen")) + args = parser.parse_args(argv) + + execution = _ModelExecutionTracker() + try: + fixture_cases = _load_cases(args.cases) + catalog, evaluated = _run_cases( + fixture_cases, + args.catalog, + mode=args.mode, + execution=execution, + embedding_backend=embedding_backend, + reranker_backend=reranker_backend, + ) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"evaluation failed: {error}", file=sys.stderr) + return 1 + + metrics = evaluate_cases(evaluated) + model_execution = execution.report(required=args.mode == "qwen") + passed = ( + metrics.auto_precision >= AUTO_PRECISION_THRESHOLD + and metrics.sensitive_field_precision >= SENSITIVE_PRECISION_THRESHOLD + and ( + args.mode != "qwen" + or ( + execution.embedding_success_count > 0 + and execution.reranker_success_count > 0 + and execution.semantic_case_count > 0 + and execution.semantic_case_pass_count == execution.semantic_case_count + ) + ) + ) + report = { + "mode": args.mode, + "catalog_version": catalog.version, + "case_count": len(evaluated), + "metrics": metrics.model_dump(mode="json"), + "model_execution": model_execution, + "gate": { + "auto_precision_threshold": AUTO_PRECISION_THRESHOLD, + "sensitive_field_precision_threshold": SENSITIVE_PRECISION_THRESHOLD, + "passed": passed, + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( + f"precision={metrics.auto_precision:.6f} " + f"coverage={metrics.coverage:.6f} " + f"sensitive_precision={metrics.sensitive_field_precision:.6f}" + ) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if passed else 2 + + +def _load_cases(path: Path) -> tuple[_FixtureCase, ...]: + cases: list[_FixtureCase] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + cases.append(_FixtureCase.model_validate_json(line)) + except ValueError as error: + raise ValueError(f"invalid case on line {line_number}: {error}") from error + return tuple(cases) + + +def _run_cases( + fixture_cases: Sequence[_FixtureCase], + catalog_path: Path, + *, + mode: str, + execution: _ModelExecutionTracker, + embedding_backend: Any | None = None, + reranker_backend: Any | None = None, +) -> tuple[Any, tuple[EvaluationCase, ...]]: + _ensure_project_root_on_path() + from app.documents.dynamic_automation.catalog import CanonicalCatalog + from app.documents.dynamic_automation.models import DocumentFieldContext + + catalog = CanonicalCatalog.load(catalog_path) + contexts = tuple( + DocumentFieldContext.model_validate(case.context.model_dump(mode="json")) + for case in fixture_cases + ) + mapper = _make_mapper( + catalog, + mode=mode, + execution=execution, + embedding_backend=embedding_backend, + reranker_backend=reranker_backend, + ) + plan = mapper.map(contexts) + evaluated: list[EvaluationCase] = [] + for fixture, mapping in zip(fixture_cases, plan.mappings, strict=True): + expected_sensitive = False + if fixture.expected_canonical_field_id is not None: + definition = catalog.get(fixture.expected_canonical_field_id) + expected_sensitive = definition.sensitivity == "sensitive" + predicted_sensitive = False + if mapping.canonical_field_id is not None: + predicted_definition = catalog.get(mapping.canonical_field_id) + predicted_sensitive = predicted_definition.sensitivity == "sensitive" + evaluated.append( + EvaluationCase( + case_id=fixture.case_id, + document_id=fixture.document_id, + expected_status=fixture.expected_status, + expected_canonical_field_id=fixture.expected_canonical_field_id, + expected_sensitive=expected_sensitive, + predicted_status=mapping.status.value, + predicted_canonical_field_id=mapping.canonical_field_id, + predicted_sensitive=predicted_sensitive, + candidate_ids=tuple( + candidate.canonical_field_id for candidate in mapping.candidates + ), + ) + ) + if ( + fixture.expected_status is EvaluationStatus.MATCHED + and mapping.evidence.rule == "semantic_decision_gate" + ): + execution.semantic_case_count += 1 + if ( + mapping.status.value == fixture.expected_status.value + and mapping.canonical_field_id == fixture.expected_canonical_field_id + ): + execution.semantic_case_pass_count += 1 + return catalog, tuple(evaluated) + + +def _make_mapper( + catalog: Any, + *, + mode: str, + execution: _ModelExecutionTracker, + embedding_backend: Any | None = None, + reranker_backend: Any | None = None, +) -> Any: + from app.documents.dynamic_automation.mapper import HybridFieldMapper, MappingThresholds + + if mode == "rule": + retriever: Any = _UnavailableRetriever() + reranker: Any = _UnavailableReranker() + min_reranker_score = 0.90 + min_margin = 0.10 + else: + from app.documents.dynamic_automation.config import DynamicAutomationSettings + from app.documents.dynamic_automation.qwen import ( + Qwen3CandidateReranker, + Qwen3EmbeddingRetriever, + ) + + settings = DynamicAutomationSettings() + if not settings.dynamic_automation_mapping_enabled: + raise ValueError( + "qwen mode requires FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED=true" + ) + retriever = _TrackingRetriever( + Qwen3EmbeddingRetriever( + settings.dynamic_automation_embedding_model_path, + backend=embedding_backend, + ), + execution, + ) + reranker = _TrackingReranker( + Qwen3CandidateReranker( + settings.dynamic_automation_reranker_model_path, + backend=reranker_backend, + definition_resolver=catalog.get, + ), + execution, + ) + min_reranker_score = settings.dynamic_automation_min_reranker_score + min_margin = settings.dynamic_automation_min_margin + return HybridFieldMapper( + catalog=catalog, + retriever=retriever, + reranker=reranker, + thresholds=MappingThresholds( + min_reranker_score=min_reranker_score, + min_margin=min_margin, + ), + top_k=5, + ) + + +@dataclass +class _ModelExecutionTracker: + embedding_success_count: int = 0 + reranker_success_count: int = 0 + semantic_case_count: int = 0 + semantic_case_pass_count: int = 0 + + def report(self, *, required: bool) -> dict[str, int | bool]: + return { + "embedding_success_count": self.embedding_success_count, + "reranker_success_count": self.reranker_success_count, + "required": required, + "semantic_case_count": self.semantic_case_count, + "semantic_case_pass_count": self.semantic_case_pass_count, + } + + +@dataclass(frozen=True) +class _TrackingRetriever: + delegate: Any + execution: _ModelExecutionTracker + + @property + def model_version(self) -> str: + return str(self.delegate.model_version) + + def retrieve(self, *args: Any, **kwargs: Any) -> Any: + result = self.delegate.retrieve(*args, **kwargs) + if isinstance(result, tuple): + self.execution.embedding_success_count += 1 + return result + + +@dataclass(frozen=True) +class _TrackingReranker: + delegate: Any + execution: _ModelExecutionTracker + + @property + def model_version(self) -> str: + return str(self.delegate.model_version) + + def rerank(self, *args: Any, **kwargs: Any) -> Any: + result = self.delegate.rerank(*args, **kwargs) + if isinstance(result, tuple): + self.execution.reranker_success_count += 1 + return result + + +class _UnavailableRetriever: + model_version = "rule-mode-no-retriever-v1" + + def retrieve(self, *_args: Any, **_kwargs: Any) -> Any: + raise RuntimeError("semantic retrieval is disabled in rule mode") + + +class _UnavailableReranker: + model_version = "rule-mode-no-reranker-v1" + + def rerank(self, *_args: Any, **_kwargs: Any) -> Any: + raise RuntimeError("semantic reranking is disabled in rule mode") + + +def _is_correct(case: EvaluationCase) -> bool: + if case.predicted_status is not case.expected_status: + return False + if case.expected_status is EvaluationStatus.MATCHED: + return case.predicted_canonical_field_id == case.expected_canonical_field_id + return case.predicted_canonical_field_id is None + + +def _ratio(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +def _ensure_project_root_on_path() -> None: + project_root = str(Path(__file__).resolve().parents[1]) + if project_root not in sys.path: + sys.path.insert(0, project_root) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/train_dynamic_mapping_models.py b/scripts/train_dynamic_mapping_models.py new file mode 100644 index 0000000..295d304 --- /dev/null +++ b/scripts/train_dynamic_mapping_models.py @@ -0,0 +1,586 @@ +"""Build deterministic domain mapping adapters from sanitized reviewer feedback.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import sys +import tempfile +import time +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from pydantic import ValidationError + +_PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(_PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(_PROJECT_ROOT)) + +from app.documents.dynamic_automation.catalog import CanonicalCatalog # noqa: E402 +from app.documents.dynamic_automation.domain_adapters import ( # noqa: E402 + adapter_file_sha256, + load_domain_embedding_retriever, + load_domain_reranker, +) +from app.documents.dynamic_automation.feedback import MappingFeedbackRecord # noqa: E402 +from app.documents.dynamic_automation.models import ( # noqa: E402 + DocumentFieldContext, + ScoredCandidate, +) +from app.documents.dynamic_automation.qwen import ( # noqa: E402 + QWEN3_EMBEDDING_CACHE_NAME, + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_CACHE_NAME, + QWEN3_RERANKER_REPO, + QWEN3_RERANKER_REVISION, + EmbeddingBackend, + LocalQwen3RerankerBackend, + LocalSentenceTransformerBackend, + RerankerBackend, +) +from app.documents.dynamic_automation.training import ( # noqa: E402 + EVALUATION_CODE_VERSION, + TRAINING_CODE_VERSION, + EvaluationMetricsEvidence, + HeldOutEvaluationReport, + ModelManifest, + TrainingSplit, + UnseenFieldEvidence, + build_hard_negatives, + build_training_split, + training_dataset_sha256, +) + + +def main( + argv: Sequence[str] | None = None, + *, + embedding_backend: EmbeddingBackend | None = None, + reranker_backend: RerankerBackend | None = None, +) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--feedback", required=True, type=Path) + parser.add_argument("--catalog", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--seed", required=True, type=int) + parser.add_argument( + "--model-kind", required=True, choices=("bi-encoder", "pair-reranker") + ) + args = parser.parse_args(argv) + + try: + if args.seed < 0: + raise ValueError("seed must be non-negative") + records = _load_feedback(args.feedback) + catalog = CanonicalCatalog.load(args.catalog) + mismatched_versions = sorted( + { + record.catalog_version + for record in records + if record.catalog_version != catalog.version + } + ) + if mismatched_versions: + raise ValueError( + f"feedback catalog_version {', '.join(mismatched_versions)} " + f"does not match loaded catalog {catalog.version}" + ) + split = build_training_split(records) + if not split.train: + raise ValueError("feedback contains no reviewer-approved training labels") + model_spec = _base_model_spec(args.model_kind) + injected_backend = ( + embedding_backend if args.model_kind == "bi-encoder" else reranker_backend + ) + if injected_backend is None: + _require_local_model( + model_spec[3], revision=model_spec[2], cache_name=model_spec[1] + ) + weights = _fit_adapter( + split, + catalog, + model_path=model_spec[3], + seed=args.seed, + model_kind=args.model_kind, + base_model_repo=model_spec[0], + base_model_revision=model_spec[2], + embedding_backend=embedding_backend, + reranker_backend=reranker_backend, + ) + dataset_sha256 = training_dataset_sha256(split) + catalog_sha256 = _file_sha256(args.catalog) + with tempfile.TemporaryDirectory(prefix="dynamic-mapping-training-") as temp_dir: + temp_root = Path(temp_dir) + artifact_path = temp_root / "adapter-weights.json" + _write_json(artifact_path, weights) + artifact_sha256 = adapter_file_sha256(artifact_path) + report = _evaluate_exported_adapter( + split, + catalog, + artifact_path=artifact_path, + artifact_sha256=artifact_sha256, + dataset_sha256=dataset_sha256, + catalog_sha256=catalog_sha256, + model_kind=args.model_kind, + embedding_backend=embedding_backend, + reranker_backend=reranker_backend, + model_path=model_spec[3], + ) + report_path = temp_root / "held-out-evaluation.json" + _write_json(report_path, report.model_dump(mode="json")) + report_sha256 = _file_sha256(report_path) + metrics = report.metrics + manifest = ModelManifest( + schema_version="dynamic-mapping-model-manifest-v2", + model_kind=( + "domain_bi_encoder" + if args.model_kind == "bi-encoder" + else "domain_pair_reranker" + ), + base_model_repo=model_spec[0], + base_model_revision=model_spec[2], + dataset_sha256=dataset_sha256, + catalog_sha256=catalog_sha256, + model_artifact_sha256=artifact_sha256, + evaluation_report_sha256=report_sha256, + catalog_version=catalog.version, + training_code_version=TRAINING_CODE_VERSION, + evaluation_code_version=EVALUATION_CODE_VERSION, + training_sample_count=len(split.train), + evaluation_sample_count=len(split.test), + training_cohort_count=len( + {example.document_layout_hash for example in split.train} + ), + evaluation_cohort_count=report.cohort_count, + auto_precision=metrics.auto_precision, + sensitive_precision=metrics.sensitive_precision, + coverage=metrics.coverage, + expected_calibration_error=metrics.expected_calibration_error, + p95_latency_ms=metrics.p95_latency_ms, + seed=args.seed, + training_canonical_field_ids=tuple( + sorted({example.canonical_field_id for example in split.train}) + ), + catalog_field_ids=tuple( + definition.field_id for definition in catalog.definitions + ), + ) + artifact_bytes = artifact_path.read_bytes() + report_bytes = report_path.read_bytes() + except ( + OSError, + RuntimeError, + ValidationError, + ValueError, + json.JSONDecodeError, + ) as error: + print(f"training failed: {error}", file=sys.stderr) + return 1 + + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "adapter-weights.json").write_bytes(artifact_bytes) + (args.output_dir / "held-out-evaluation.json").write_bytes(report_bytes) + _write_json( + args.output_dir / "model-manifest.json", manifest.model_dump(mode="json") + ) + print( + f"trained={manifest.model_kind} dataset_sha256={manifest.dataset_sha256} " + f"catalog_sha256={manifest.catalog_sha256}" + ) + return 0 + + +def _load_feedback(path: Path) -> tuple[MappingFeedbackRecord, ...]: + records: list[MappingFeedbackRecord] = [] + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not line.strip(): + continue + try: + records.append(MappingFeedbackRecord.model_validate_json(line)) + except ValidationError as error: + raise ValueError(f"invalid feedback on line {line_number}: {error}") from error + return tuple(records) + + +def _base_model_spec(model_kind: str) -> tuple[str, str, str, Path]: + if model_kind == "bi-encoder": + repo = QWEN3_EMBEDDING_REPO + cache_name = QWEN3_EMBEDDING_CACHE_NAME + revision = QWEN3_EMBEDDING_REVISION + explicit_path = os.environ.get("FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH") + else: + repo = QWEN3_RERANKER_REPO + cache_name = QWEN3_RERANKER_CACHE_NAME + revision = QWEN3_RERANKER_REVISION + explicit_path = os.environ.get("FOWOCO_DYNAMIC_AUTOMATION_RERANKER_MODEL_PATH") + if explicit_path: + path = Path(explicit_path) + else: + cache_root = Path( + os.environ.get( + "FOWOCO_MODEL_CACHE_DIR", + str(Path(tempfile.gettempdir()) / "fowoco-model-cache"), + ) + ) + path = cache_root / cache_name / revision + return repo, cache_name, revision, path + + +def _require_local_model(path: Path, *, revision: str, cache_name: str) -> None: + if tuple(path.parts[-2:]) != (cache_name, revision): + raise ValueError( + f"base model path must end with pinned cache path {cache_name}/{revision}" + ) + if not path.is_dir() or not (path / "config.json").is_file(): + raise ValueError( + f"pinned local base model cache is unavailable at {path}; " + "provision it explicitly before training" + ) + has_weights = any(path.glob("*.safetensors")) or any(path.glob("*.bin")) + if not has_weights: + raise ValueError(f"pinned local base model cache has no model weights at {path}") + + +def _fit_adapter( + split: TrainingSplit, + catalog: CanonicalCatalog, + *, + model_path: Path, + seed: int, + model_kind: str, + base_model_repo: str, + base_model_revision: str, + embedding_backend: EmbeddingBackend | None = None, + reranker_backend: RerankerBackend | None = None, +) -> dict[str, Any]: + common = { + "format_version": "dynamic-mapping-adapter-v2", + "model_kind": model_kind, + "base_model_repo": base_model_repo, + "base_model_revision": base_model_revision, + "seed": seed, + } + if model_kind == "bi-encoder": + common["weights"] = _fit_bi_encoder( + split, + catalog, + model_path=model_path, + backend=embedding_backend, + ) + else: + common["weights"] = _fit_pair_reranker( + split, + catalog, + model_path=model_path, + backend=reranker_backend, + ) + return common + + +def _fit_bi_encoder( + split: TrainingSplit, + catalog: CanonicalCatalog, + *, + model_path: Path, + backend: EmbeddingBackend | None = None, +) -> dict[str, Any]: + backend = backend or LocalSentenceTransformerBackend(model_path) + queries = tuple(example.query_text for example in split.train) + documents = tuple( + _definition_text(catalog, example.canonical_field_id) + for example in split.train + ) + query_vectors = backend.encode_queries(queries, max_length=512, batch_size=8) + document_vectors = backend.encode_documents( + documents, max_length=512, batch_size=8 + ) + if len(query_vectors) != len(split.train) or len(document_vectors) != len( + split.train + ): + raise RuntimeError("invalid embedding batch size during adapter training") + if not query_vectors or not query_vectors[0]: + raise RuntimeError("base embedding model returned empty vectors") + dimensions = {len(vector) for vector in (*query_vectors, *document_vectors)} + if len(dimensions) != 1: + raise RuntimeError("base embedding model returned inconsistent dimensions") + pair_count = len(query_vectors) + query_bias = tuple( + sum(document[index] - query[index] for query, document in zip( + query_vectors, document_vectors, strict=True + )) + / pair_count + for index in range(len(query_vectors[0])) + ) + return { + "adapter_kind": "query_bias_projection", + "embedding_dimension": len(query_bias), + "positive_pair_count": pair_count, + "query_bias": query_bias, + } + + +def _fit_pair_reranker( + split: TrainingSplit, + catalog: CanonicalCatalog, + *, + model_path: Path, + backend: RerankerBackend | None = None, +) -> dict[str, Any]: + backend = backend or LocalQwen3RerankerBackend(model_path) + positive_pairs = tuple( + (example.query_text, _definition_text(catalog, example.canonical_field_id)) + for example in split.train + ) + negatives = build_hard_negatives(split, catalog) + negative_pairs = tuple( + (pair.query_text, _definition_text(catalog, pair.negative_canonical_field_id)) + for pair in negatives + ) + if not negative_pairs: + raise ValueError("pair-reranker training requires type-compatible negatives") + positive_scores = backend.score_pairs( + positive_pairs, max_length=512, batch_size=2 + ) + negative_scores = backend.score_pairs( + negative_pairs, max_length=512, batch_size=2 + ) + if len(positive_scores) != len(positive_pairs) or len(negative_scores) != len( + negative_pairs + ): + raise RuntimeError("invalid reranker batch size during adapter training") + positive_mean = sum(positive_scores) / len(positive_scores) + negative_mean = sum(negative_scores) / len(negative_scores) + threshold = (positive_mean + negative_mean) / 2 + separation = positive_mean - negative_mean + scale = 1 / (separation if abs(separation) >= 1e-6 else 1e-6) + return { + "adapter_kind": "score_calibration", + "positive_pair_count": len(positive_pairs), + "negative_pair_count": len(negative_pairs), + "scale": scale, + "bias": -threshold * scale, + } + + +def _evaluate_exported_adapter( + split: TrainingSplit, + catalog: CanonicalCatalog, + *, + artifact_path: Path, + artifact_sha256: str, + dataset_sha256: str, + catalog_sha256: str, + model_kind: str, + embedding_backend: EmbeddingBackend | None, + reranker_backend: RerankerBackend | None, + model_path: Path, +) -> HeldOutEvaluationReport: + if not split.test: + raise ValueError("training requires at least one held-out evaluation cohort") + if model_kind == "bi-encoder": + adapter: Any = load_domain_embedding_retriever( + artifact_path, + backend=embedding_backend, + model_path=model_path, + expected_sha256=artifact_sha256, + ) + else: + adapter = load_domain_reranker( + artifact_path, + backend=reranker_backend, + model_path=model_path, + definition_resolver=catalog.get, + expected_sha256=artifact_sha256, + ) + + correct = 0 + predictions = 0 + sensitive_correct = 0 + sensitive_predictions = 0 + calibration_errors: list[float] = [] + latencies_ms: list[float] = [] + for example in split.test: + definition = catalog.get(example.canonical_field_id) + context = _context_for_training_example(example, definition.compatible_field_types[0]) + candidates = catalog.compatible(context) + started = time.perf_counter() + ranked = _rank_with_adapter( + adapter, + model_kind=model_kind, + context=context, + candidates=candidates, + ) + latencies_ms.append((time.perf_counter() - started) * 1_000) + if not ranked: + continue + predictions += 1 + is_correct = ranked[0].canonical_field_id == example.canonical_field_id + correct += int(is_correct) + calibration_errors.append(abs(ranked[0].score - float(is_correct))) + if definition.sensitivity == "sensitive": + sensitive_predictions += 1 + sensitive_correct += int(is_correct) + + training_ids = {example.canonical_field_id for example in split.train} + unseen_definition = next( + definition + for definition in catalog.definitions + if definition.field_id not in training_ids + ) + unseen_context = _generated_unseen_context(unseen_definition) + unseen_ranked = _rank_with_adapter( + adapter, + model_kind=model_kind, + context=unseen_context, + candidates=catalog.compatible(unseen_context), + ) + unseen_ids = tuple(item.canonical_field_id for item in unseen_ranked[:20]) + try: + unseen_rank: int | None = unseen_ids.index(unseen_definition.field_id) + 1 + except ValueError: + unseen_rank = None + query_payload = json.dumps( + unseen_context.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + sample_count = len(split.test) + metrics = EvaluationMetricsEvidence( + auto_precision=_ratio(correct, predictions), + sensitive_precision=( + _ratio(sensitive_correct, sensitive_predictions) + if sensitive_predictions + else 1.0 + ), + coverage=_ratio(predictions, sample_count), + expected_calibration_error=( + sum(calibration_errors) / len(calibration_errors) + if calibration_errors + else 1.0 + ), + p95_latency_ms=_percentile_95(latencies_ms), + ) + return HeldOutEvaluationReport( + schema_version="dynamic-mapping-held-out-v2", + evaluation_code_version=EVALUATION_CODE_VERSION, + model_artifact_sha256=artifact_sha256, + dataset_sha256=dataset_sha256, + catalog_sha256=catalog_sha256, + catalog_version=catalog.version, + sample_count=sample_count, + cohort_count=len({example.document_layout_hash for example in split.test}), + model_execution_count=sample_count + 1, + metrics=metrics, + unseen_field_evidence=UnseenFieldEvidence( + case_id=f"generated-unseen:{unseen_definition.field_id}", + canonical_field_id=unseen_definition.field_id, + query_sha256=hashlib.sha256(query_payload).hexdigest(), + candidate_ids=unseen_ids, + retrieved_rank=unseen_rank, + ), + ) + + +def _rank_with_adapter( + adapter: Any, + *, + model_kind: str, + context: DocumentFieldContext, + candidates: Sequence[Any], +) -> tuple[ScoredCandidate, ...]: + if model_kind == "bi-encoder": + return adapter.retrieve(context, candidates, min(20, len(candidates))) + initial = tuple( + ScoredCandidate(canonical_field_id=item.field_id, score=0.5, rank=index) + for index, item in enumerate(candidates[:20], start=1) + ) + return adapter.rerank(context, initial) + + +def _context_for_training_example(example: Any, field_type: str) -> DocumentFieldContext: + lines = example.query_text.splitlines() + label = lines[0].removeprefix("label: ") if lines else example.field_id + section = lines[1].removeprefix("section: ") if len(lines) > 1 else "" + return DocumentFieldContext( + field_id=example.field_id, + container_id="held-out-evaluation", + label=label[:200], + normalized_label=label.casefold().replace(" ", "")[:200], + field_type=field_type, + document_title="Held-out mapping evaluation", + section=section[:200], + row_labels=(section[:200], label[:200]) if section else (label[:200],), + nearby_labels=(), + options=(), + repeat_index=example.repeat_index, + required=True, + kind="text_field", + ) + + +def _generated_unseen_context(definition: Any) -> DocumentFieldContext: + label = definition.description[:200] + return DocumentFieldContext( + field_id=f"generated-{definition.field_id}"[:200], + container_id="generated-unseen-evaluation", + label=label, + normalized_label=label.casefold().replace(" ", "")[:200], + field_type=definition.compatible_field_types[0], + document_title="Generated unseen catalog evaluation", + section=definition.entity[:200], + row_labels=(definition.entity[:200], label), + nearby_labels=tuple(alias[:200] for alias in definition.aliases[:4]), + options=(), + repeat_index=0, + required=True, + kind="text_field", + ) + + +def _percentile_95(values: Sequence[float]) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = max(0, math.ceil(len(ordered) * 0.95) - 1) + return ordered[index] + + +def _ratio(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +def _definition_text(catalog: CanonicalCatalog, field_id: str) -> str: + definition = catalog.get(field_id) + return "\n".join( + ( + f"field id: {definition.field_id}", + f"entity: {definition.entity}", + f"value type: {definition.value_type}", + f"aliases: {' | '.join(definition.aliases)}", + f"description: {definition.description}", + ) + ) + + +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _write_json(path: Path, payload: Any) -> None: + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/documents/dynamic_automation/fakes.py b/tests/documents/dynamic_automation/fakes.py new file mode 100644 index 0000000..f90bd5f --- /dev/null +++ b/tests/documents/dynamic_automation/fakes.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from app.documents.dynamic_automation.models import ( + CanonicalFieldDefinition, + DocumentFieldContext, + ScoredCandidate, +) + + +@dataclass +class FakeCandidateRetriever: + results: tuple[ScoredCandidate, ...] | None = () + model_version: str = "fake-embedding-v1" + error: Exception | None = None + enforce_top_k: bool = True + + def retrieve( + self, + context: DocumentFieldContext, + candidates: Sequence[CanonicalFieldDefinition], + top_k: int, + ) -> tuple[ScoredCandidate, ...] | None: + del context, candidates + if self.error is not None: + raise self.error + if self.results is None: + return None + return self.results[:top_k] if self.enforce_top_k else self.results + + +@dataclass +class FakeCandidateReranker: + results: tuple[ScoredCandidate, ...] | None = () + model_version: str = "fake-reranker-v1" + error: Exception | None = None + + def rerank( + self, + context: DocumentFieldContext, + candidates: Sequence[ScoredCandidate], + ) -> tuple[ScoredCandidate, ...] | None: + del context, candidates + if self.error is not None: + raise self.error + return self.results + + +def scored(*items: tuple[str, float]) -> tuple[ScoredCandidate, ...]: + return tuple( + ScoredCandidate(canonical_field_id=field_id, score=score, rank=rank) + for rank, (field_id, score) in enumerate(items, start=1) + ) diff --git a/tests/documents/dynamic_automation/test_catalog.py b/tests/documents/dynamic_automation/test_catalog.py new file mode 100644 index 0000000..e53f409 --- /dev/null +++ b/tests/documents/dynamic_automation/test_catalog.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.models import DocumentFieldContext + +DEFAULT_CATALOG_PATH = ( + Path(__file__).parents[3] + / "app" + / "documents" + / "dynamic_automation" + / "resources" + / "canonical_fields.v1.yaml" +) + +VALID_FIELD = """ +version: v1 +fields: + - field_id: company.phone + entity: company + value_type: phone + aliases: [전화번호, Company phone] + description: Company contact telephone number. + compatible_field_types: [phone, text] + source: + view: document_company_view + column: phone + scope_keys: [tenant_id, company_id] + sensitivity: business + formatter: phone +""" + + +def _context(*, field_type: str = "phone", repeat_index: int = 0) -> DocumentFieldContext: + return DocumentFieldContext( + field_id="field-1", + label="전화번호", + normalized_label="전화번호", + field_type=field_type, + document_title="통합신청서", + section="현재 근무처", + row_labels=("현재 근무처", "전화번호"), + nearby_labels=(), + options=(), + repeat_index=repeat_index, + required=True, + kind="text_field", + ) + + +def test_catalog_rejects_duplicate_ids_and_unapproved_identifiers(tmp_path: Path) -> None: + path = tmp_path / "catalog.yaml" + path.write_text( + VALID_FIELD + + """ + - field_id: company.phone + entity: company + value_type: phone + aliases: [전화번호] + description: Duplicate. + compatible_field_types: [phone] + source: + view: document_company_view + column: phone + scope_keys: [tenant_id, company_id] + sensitivity: business + formatter: phone +""", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="duplicate"): + CanonicalCatalog.load(path) + + path.write_text( + VALID_FIELD.replace("document_company_view", "document-company-view"), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="identifier"): + CanonicalCatalog.load(path) + + +def test_catalog_returns_known_field_and_rejects_unknown_field() -> None: + catalog = CanonicalCatalog.load(DEFAULT_CATALOG_PATH) + + assert catalog.get("company.phone").source.column == "phone" + with pytest.raises(KeyError, match="unknown canonical field"): + catalog.get("company.unknown") + + +def test_catalog_definitions_are_immutable_and_stably_iterable() -> None: + catalog = CanonicalCatalog.load(DEFAULT_CATALOG_PATH) + + field_ids = tuple(definition.field_id for definition in catalog.definitions) + assert field_ids == tuple(sorted(field_ids)) + assert tuple(catalog) == catalog.definitions + + with pytest.raises(TypeError): + catalog._fields_by_id["fabricated.field"] = catalog.definitions[0] # type: ignore[index] + with pytest.raises(ValidationError): + catalog.definitions[0].field_id = "fabricated.field" # type: ignore[misc] + + +def test_compatible_filters_wrong_type_without_repeat_index_eligibility() -> None: + catalog = CanonicalCatalog.load(DEFAULT_CATALOG_PATH) + + compatible_ids = {item.field_id for item in catalog.compatible(_context())} + assert "company.phone" in compatible_ids + assert "worker.date_of_birth" not in compatible_ids + + repeated_ids = {item.field_id for item in catalog.compatible(_context(repeat_index=1))} + assert "company.phone" in repeated_ids + + +def test_document_field_context_rejects_oversized_labels_and_options() -> None: + with pytest.raises(ValidationError): + DocumentFieldContext(**{**_context().model_dump(), "row_labels": ("x" * 201,)}) + + with pytest.raises(ValidationError): + DocumentFieldContext(**{**_context().model_dump(), "nearby_labels": ("x" * 201,)}) + + with pytest.raises(ValidationError): + DocumentFieldContext( + **{**_context().model_dump(), "options": tuple("option" for _ in range(51))} + ) + + with pytest.raises(ValidationError): + DocumentFieldContext(**{**_context().model_dump(), "options": ("x" * 201,)}) diff --git a/tests/documents/dynamic_automation/test_domain_adapters.py b/tests/documents/dynamic_automation/test_domain_adapters.py new file mode 100644 index 0000000..330a8f7 --- /dev/null +++ b/tests/documents/dynamic_automation/test_domain_adapters.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.domain_adapters import ( + adapter_file_sha256, + load_domain_embedding_retriever, + load_domain_reranker, +) +from app.documents.dynamic_automation.feedback import MappingFeedbackRecord +from app.documents.dynamic_automation.models import DocumentFieldContext, ScoredCandidate +from app.documents.dynamic_automation.qwen import ( + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_REPO, + QWEN3_RERANKER_REVISION, +) +from app.documents.dynamic_automation.training import TrainingExample, TrainingSplit +from scripts import compare_dynamic_mapping_models as comparison_cli +from scripts import train_dynamic_mapping_models as training_cli + +ROOT = Path(__file__).parents[3] +CATALOG_PATH = ( + ROOT / "app/documents/dynamic_automation/resources/canonical_fields.v1.yaml" +) + + +class ProjectionBaseBackend: + """Complete fixed embedding backend with deliberately wrong raw worker-name queries.""" + + def encode_queries( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + del max_length, batch_size + return tuple( + (-1.0, 2.0) if "nationality" in text.casefold() else (0.0, 1.0) + for text in texts + ) + + def encode_documents( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + del max_length, batch_size + vectors: list[tuple[float, ...]] = [] + for text in texts: + if "worker.legal_name" in text: + vectors.append((1.0, 0.0)) + elif "worker.nationality" in text: + vectors.append((0.0, 1.0)) + else: + vectors.append((-1.0, 0.0)) + return tuple(vectors) + + +class CalibrationBaseBackend: + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: + del max_length, batch_size + return tuple(0.8 if "company.phone" in definition else 0.2 for _, definition in pairs) + + +class ReversedCalibrationBackend: + """Complete backend whose raw ranking is the opposite of its training labels.""" + + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: + del max_length, batch_size + return tuple( + 0.1 if "worker.phone" in definition else 0.9 + for _, definition in pairs + ) + + +def context(*, label: str = "Worker name", field_type: str = "text") -> DocumentFieldContext: + return DocumentFieldContext( + field_id="field-1", + container_id="section0.table0", + label=label, + normalized_label=label.casefold().replace(" ", ""), + field_type=field_type, + document_title="Application", + section="Worker", + row_labels=("Worker", label), + nearby_labels=(), + options=(), + repeat_index=0, + required=True, + kind="text_field", + ) + + +def write_artifact(path: Path, payload: dict[str, object]) -> str: + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_embedding_projection_artifact_is_loaded_and_changes_retrieval( + tmp_path: Path, +) -> None: + catalog = CanonicalCatalog.load(CATALOG_PATH) + artifact_path = tmp_path / "embedding-adapter.json" + expected_hash = write_artifact( + artifact_path, + { + "format_version": "dynamic-mapping-adapter-v2", + "model_kind": "bi-encoder", + "base_model_repo": QWEN3_EMBEDDING_REPO, + "base_model_revision": QWEN3_EMBEDDING_REVISION, + "seed": 42, + "weights": { + "adapter_kind": "query_bias_projection", + "embedding_dimension": 2, + "positive_pair_count": 1, + "query_bias": [1.0, -1.0], + }, + }, + ) + + retriever = load_domain_embedding_retriever( + artifact_path, + backend=ProjectionBaseBackend(), + expected_sha256=expected_hash, + ) + ranked = retriever.retrieve(context(), catalog.compatible(context()), top_k=3) + + assert ranked[0].canonical_field_id == "worker.legal_name" + assert retriever.model_version.endswith(f"@{expected_hash}") + assert adapter_file_sha256(artifact_path) == expected_hash + + with pytest.raises(ValueError, match="SHA-256"): + load_domain_embedding_retriever( + artifact_path, + backend=ProjectionBaseBackend(), + expected_sha256="f" * 64, + ) + + +def test_reranker_calibration_artifact_is_loaded_through_reranker_port( + tmp_path: Path, +) -> None: + catalog = CanonicalCatalog.load(CATALOG_PATH) + artifact_path = tmp_path / "reranker-adapter.json" + expected_hash = write_artifact( + artifact_path, + { + "format_version": "dynamic-mapping-adapter-v2", + "model_kind": "pair-reranker", + "base_model_repo": QWEN3_RERANKER_REPO, + "base_model_revision": QWEN3_RERANKER_REVISION, + "seed": 42, + "weights": { + "adapter_kind": "score_calibration", + "positive_pair_count": 1, + "negative_pair_count": 1, + "scale": 2.0, + "bias": -1.0, + }, + }, + ) + reranker = load_domain_reranker( + artifact_path, + backend=CalibrationBaseBackend(), + definition_resolver=catalog.get, + expected_sha256=expected_hash, + ) + candidates = ( + ScoredCandidate(canonical_field_id="worker.phone", score=0.5, rank=1), + ScoredCandidate(canonical_field_id="company.phone", score=0.5, rank=2), + ) + + ranked = reranker.rerank(context(label="Company contact", field_type="phone"), candidates) + + assert ranked[0].canonical_field_id == "company.phone" + assert 0 <= ranked[0].score <= 1 + assert reranker.model_version.endswith(f"@{expected_hash}") + + +def test_trained_reranker_calibration_can_reverse_a_wrong_base_ranking( + tmp_path: Path, +) -> None: + catalog = CanonicalCatalog.load(CATALOG_PATH) + split = TrainingSplit( + train=( + TrainingExample( + document_layout_hash="a" * 64, + document_kind="application", + document_version="v1", + source_institution="institution-a", + field_context_hash="b" * 64, + field_id="worker-phone", + repeat_index=0, + query_text="Worker contact number", + canonical_field_id="worker.phone", + catalog_version="v1", + ), + ), + test=(), + ) + weights = training_cli._fit_pair_reranker( + split, + catalog, + model_path=tmp_path / "unused-model", + backend=ReversedCalibrationBackend(), + ) + artifact_path = tmp_path / "trained-reranker-adapter.json" + expected_hash = write_artifact( + artifact_path, + { + "format_version": "dynamic-mapping-adapter-v2", + "model_kind": "pair-reranker", + "base_model_repo": QWEN3_RERANKER_REPO, + "base_model_revision": QWEN3_RERANKER_REVISION, + "seed": 42, + "weights": weights, + }, + ) + reranker = load_domain_reranker( + artifact_path, + backend=ReversedCalibrationBackend(), + definition_resolver=catalog.get, + expected_sha256=expected_hash, + ) + + ranked = reranker.rerank( + context(label="Worker contact number", field_type="phone"), + ( + ScoredCandidate(canonical_field_id="company.phone", score=0.9, rank=1), + ScoredCandidate(canonical_field_id="worker.phone", score=0.1, rank=2), + ), + ) + + assert weights["scale"] < 0 + assert ranked[0].canonical_field_id == "worker.phone" + + +def feedback_payload(*, layout_hash: str, field_id: str) -> dict[str, object]: + return { + "schema_version": "v2", + "layout_hash": layout_hash, + "document_kind": f"kind-{field_id}", + "document_version": f"version-{field_id}", + "source_institution": f"institution-{field_id}", + "field_context_hash": hashlib.sha256(field_id.encode("utf-8")).hexdigest(), + "field_id": field_id, + "repeat_index": 0, + "label": "Worker name", + "section": "Worker", + "row_labels": ["Worker", "Worker name"], + "nearby_labels": [], + "predicted_status": "MATCHED", + "predicted_canonical_field_id": "worker.legal_name", + "final_canonical_field_id": "worker.legal_name", + "decision": "accepted", + "candidate_scores": [ + {"canonical_field_id": "worker.legal_name", "score": 0.99, "rank": 1} + ], + "catalog_version": "v1", + "model_version": "fake-base-v1", + } + + +def test_fake_backend_train_export_load_evaluate_and_compare_cli( + tmp_path: Path, +) -> None: + feedback_path = tmp_path / "feedback.jsonl" + records = ( + MappingFeedbackRecord.model_validate( + feedback_payload(layout_hash="a" * 64, field_id="worker-name-a") + ), + MappingFeedbackRecord.model_validate( + feedback_payload(layout_hash="b" * 64, field_id="worker-name-b") + ), + ) + feedback_path.write_text( + "".join(record.model_dump_json() + "\n" for record in records), + encoding="utf-8", + ) + candidate_dir = tmp_path / "candidate" + + exit_code = training_cli.main( + [ + "--feedback", + str(feedback_path), + "--catalog", + str(CATALOG_PATH), + "--output-dir", + str(candidate_dir), + "--seed", + "42", + "--model-kind", + "bi-encoder", + ], + embedding_backend=ProjectionBaseBackend(), + ) + + assert exit_code == 0 + artifact_path = candidate_dir / "adapter-weights.json" + report_path = candidate_dir / "held-out-evaluation.json" + candidate_manifest_path = candidate_dir / "model-manifest.json" + candidate_manifest = json.loads(candidate_manifest_path.read_text(encoding="utf-8")) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert candidate_manifest["model_artifact_sha256"] == adapter_file_sha256( + artifact_path + ) + assert candidate_manifest["evaluation_report_sha256"] == hashlib.sha256( + report_path.read_bytes() + ).hexdigest() + assert report["unseen_field_evidence"]["retrieved_rank"] is not None + assert "unseen_catalog_retrieved" not in candidate_manifest + + catalog = CanonicalCatalog.load(CATALOG_PATH) + loaded = load_domain_embedding_retriever( + artifact_path, + backend=ProjectionBaseBackend(), + expected_sha256=candidate_manifest["model_artifact_sha256"], + ) + assert loaded.retrieve(context(), catalog.compatible(context()), top_k=3)[ + 0 + ].canonical_field_id == "worker.legal_name" + + baseline_dir = tmp_path / "baseline" + baseline_dir.mkdir() + baseline_artifact = baseline_dir / "adapter-weights.json" + baseline_artifact.write_bytes(artifact_path.read_bytes()) + baseline_report = json.loads(report_path.read_text(encoding="utf-8")) + baseline_report["metrics"]["p95_latency_ms"] += 100.0 + baseline_report_path = baseline_dir / "held-out-evaluation.json" + baseline_report_path.write_text( + json.dumps(baseline_report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + baseline_manifest = {**candidate_manifest, "model_kind": "qwen_baseline"} + baseline_manifest["p95_latency_ms"] = baseline_report["metrics"]["p95_latency_ms"] + baseline_manifest["evaluation_report_sha256"] = hashlib.sha256( + baseline_report_path.read_bytes() + ).hexdigest() + baseline_manifest_path = baseline_dir / "model-manifest.json" + baseline_manifest_path.write_text( + json.dumps(baseline_manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + decision_path = tmp_path / "decision.json" + + compare_exit = comparison_cli.main( + [ + "--baseline", + str(baseline_manifest_path), + "--baseline-artifact", + str(baseline_artifact), + "--baseline-report", + str(baseline_report_path), + "--candidate", + str(candidate_manifest_path), + "--candidate-artifact", + str(artifact_path), + "--candidate-report", + str(report_path), + "--output", + str(decision_path), + ] + ) + + assert compare_exit == 0 + assert json.loads(decision_path.read_text(encoding="utf-8"))["promote"] is True + + report_path.write_text(report_path.read_text(encoding="utf-8") + " ", encoding="utf-8") + assert comparison_cli.main( + [ + "--baseline", + str(baseline_manifest_path), + "--baseline-artifact", + str(baseline_artifact), + "--baseline-report", + str(baseline_report_path), + "--candidate", + str(candidate_manifest_path), + "--candidate-artifact", + str(artifact_path), + "--candidate-report", + str(report_path), + "--output", + str(decision_path), + ] + ) == 2 diff --git a/tests/documents/dynamic_automation/test_evaluation.py b/tests/documents/dynamic_automation/test_evaluation.py new file mode 100644 index 0000000..82e77c5 --- /dev/null +++ b/tests/documents/dynamic_automation/test_evaluation.py @@ -0,0 +1,570 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from collections.abc import Sequence +from pathlib import Path +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +import scripts.evaluate_dynamic_mapping as evaluation +from scripts.evaluate_dynamic_mapping import EvaluationCase, evaluate_cases + +ROOT = Path(__file__).parents[3] +CASES_PATH = ROOT / "tests" / "fixtures" / "dynamic_automation" / "mapping_cases.jsonl" +CATALOG_PATH = ( + ROOT + / "app" + / "documents" + / "dynamic_automation" + / "resources" + / "canonical_fields.v1.yaml" +) + + +class SemanticEmbeddingBackend: + """Complete deterministic stand-in for the unavailable local Qwen encoder.""" + + def __init__(self) -> None: + self.query_batches = 0 + self.document_batches = 0 + + def encode_queries( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + assert max_length == 512 + assert batch_size == 8 + self.query_batches += 1 + return tuple((1.0, 0.0) for _ in texts) + + def encode_documents( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + assert max_length == 512 + assert batch_size == 8 + self.document_batches += 1 + return tuple( + (1.0, 0.0) if "canonical field: company.phone" in text else (0.0, 1.0) + for text in texts + ) + + +class SemanticRerankerBackend: + """Complete deterministic stand-in for the unavailable local Qwen reranker.""" + + def __init__(self) -> None: + self.pair_batches = 0 + + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: + assert max_length == 512 + assert batch_size == 8 + self.pair_batches += 1 + scores: list[float] = [] + for query, definition in pairs: + if "Employer contact line" in query: + scores.append(0.99 if "company.phone" in definition else 0.20) + else: + scores.append(0.75 if "company.phone" in definition else 0.74) + return tuple(scores) + + +def matched_case(*, correct: bool) -> EvaluationCase: + return EvaluationCase( + case_id="matched", + document_id="document-1", + expected_status="MATCHED", + expected_canonical_field_id="worker.phone", + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id=("worker.phone" if correct else "company.phone"), + candidate_ids=("company.phone", "worker.phone"), + ) + + +def ambiguous_case(*, correct: bool) -> EvaluationCase: + return EvaluationCase( + case_id="ambiguous", + document_id="document-2", + expected_status="AMBIGUOUS", + expected_canonical_field_id=None, + expected_sensitive=False, + predicted_status="AMBIGUOUS" if correct else "MATCHED", + predicted_canonical_field_id=None if correct else "worker.phone", + candidate_ids=(), + ) + + +def test_selective_metrics_count_wrong_auto_match() -> None: + metrics = evaluate_cases([matched_case(correct=False), ambiguous_case(correct=True)]) + + assert metrics.auto_precision == 0.0 + assert metrics.coverage == 0.5 + + +def test_auto_precision_counts_false_match_on_expected_non_data() -> None: + correct_sensitive = EvaluationCase( + case_id="correct-sensitive", + document_id="document-1", + expected_status="MATCHED", + expected_canonical_field_id="identity.passport_number", + expected_sensitive=True, + predicted_status="MATCHED", + predicted_canonical_field_id="identity.passport_number", + candidate_ids=("identity.passport_number",), + ) + false_non_data = EvaluationCase( + case_id="false-non-data", + document_id="document-2", + expected_status="NON_DATA", + expected_canonical_field_id=None, + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="company.phone", + candidate_ids=("company.phone",), + ) + + metrics = evaluate_cases([correct_sensitive, false_non_data]) + + assert metrics.auto_precision == 0.5 + assert metrics.coverage == 1.0 + + +def test_cli_exits_two_when_auto_precision_is_below_threshold( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cases = ( + EvaluationCase( + case_id="correct-sensitive", + document_id="document-1", + expected_status="MATCHED", + expected_canonical_field_id="identity.passport_number", + expected_sensitive=True, + predicted_status="MATCHED", + predicted_canonical_field_id="identity.passport_number", + candidate_ids=("identity.passport_number",), + ), + EvaluationCase( + case_id="false-non-data", + document_id="document-2", + expected_status="NON_DATA", + expected_canonical_field_id=None, + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="company.phone", + candidate_ids=("company.phone",), + ), + ) + cases_path = tmp_path / "cases.jsonl" + cases_path.write_text("", encoding="utf-8") + output_path = tmp_path / "report.json" + monkeypatch.setattr( + evaluation, + "_run_cases", + lambda *_args, **_kwargs: (SimpleNamespace(version="v1"), cases), + ) + + exit_code = evaluation.main( + [ + "--cases", + str(cases_path), + "--catalog", + "unused.yaml", + "--mode", + "rule", + "--output", + str(output_path), + ] + ) + + assert exit_code == 2 + assert json.loads(output_path.read_text(encoding="utf-8"))["gate"]["passed"] is False + + +def test_sensitive_precision_counts_false_assignment_into_sensitive_target() -> None: + correct_non_sensitive = EvaluationCase( + case_id="correct-company-phone", + document_id="document-1", + expected_status="MATCHED", + expected_canonical_field_id="company.phone", + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="company.phone", + predicted_sensitive=False, + candidate_ids=("company.phone",), + ) + false_sensitive_target = EvaluationCase( + case_id="wrong-passport-target", + document_id="document-2", + expected_status="MATCHED", + expected_canonical_field_id="company.phone", + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="identity.passport_number", + predicted_sensitive=True, + candidate_ids=("identity.passport_number", "company.phone"), + ) + + metrics = evaluate_cases([correct_non_sensitive, false_sensitive_target]) + + assert metrics.auto_precision == 0.5 + assert metrics.sensitive_field_precision == 0.0 + + +def test_cli_exits_two_when_sensitive_precision_is_below_threshold( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + correct = tuple( + EvaluationCase( + case_id=f"correct-{index}", + document_id=f"document-{index}", + expected_status="MATCHED", + expected_canonical_field_id="company.phone", + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="company.phone", + predicted_sensitive=False, + candidate_ids=("company.phone",), + ) + for index in range(100) + ) + false_sensitive_target = EvaluationCase( + case_id="wrong-passport-target", + document_id="document-sensitive-error", + expected_status="MATCHED", + expected_canonical_field_id="company.phone", + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="identity.passport_number", + predicted_sensitive=True, + candidate_ids=("identity.passport_number", "company.phone"), + ) + cases_path = tmp_path / "cases.jsonl" + cases_path.write_text("", encoding="utf-8") + output_path = tmp_path / "report.json" + monkeypatch.setattr( + evaluation, + "_run_cases", + lambda *_args, **_kwargs: ( + SimpleNamespace(version="v1"), + (*correct, false_sensitive_target), + ), + ) + + exit_code = evaluation.main( + [ + "--cases", + str(cases_path), + "--catalog", + "unused.yaml", + "--mode", + "rule", + "--output", + str(output_path), + ] + ) + + report = json.loads(output_path.read_text(encoding="utf-8")) + assert exit_code == 2 + assert report["metrics"]["auto_precision"] == 0.9900990099009901 + assert report["metrics"]["sensitive_field_precision"] == 0.0 + assert report["gate"]["passed"] is False + + +def test_metrics_are_derived_from_literal_case_outcomes() -> None: + cases = [ + EvaluationCase( + case_id="correct-auto", + document_id="document-1", + expected_status="MATCHED", + expected_canonical_field_id="identity.passport_number", + expected_sensitive=True, + predicted_status="MATCHED", + predicted_canonical_field_id="identity.passport_number", + candidate_ids=("identity.passport_number",), + ), + EvaluationCase( + case_id="wrong-auto", + document_id="document-1", + expected_status="MATCHED", + expected_canonical_field_id="worker.phone", + expected_sensitive=False, + predicted_status="MATCHED", + predicted_canonical_field_id="company.phone", + candidate_ids=("company.phone", "worker.phone"), + ), + ambiguous_case(correct=True), + EvaluationCase( + case_id="non-data", + document_id="document-2", + expected_status="NON_DATA", + expected_canonical_field_id=None, + expected_sensitive=False, + predicted_status="NON_DATA", + predicted_canonical_field_id=None, + candidate_ids=(), + ), + EvaluationCase( + case_id="deferred-sensitive", + document_id="document-3", + expected_status="MATCHED", + expected_canonical_field_id="identity.alien_registration_number", + expected_sensitive=True, + predicted_status="AMBIGUOUS", + predicted_canonical_field_id=None, + candidate_ids=("identity.alien_registration_number",), + ), + ] + + metrics = evaluate_cases(cases, top_k=2) + + assert metrics.extraction_precision == 1.0 + assert metrics.extraction_recall == 1.0 + assert metrics.top_1_accuracy == pytest.approx(2 / 3) + assert metrics.top_k_recall == 1.0 + assert metrics.auto_precision == 0.5 + assert metrics.coverage == 0.5 + assert metrics.ambiguous_accuracy == 1.0 + assert metrics.sensitive_field_precision == 1.0 + assert metrics.document_zero_error_rate == pytest.approx(1 / 3) + + +def test_empty_evaluation_is_deterministic() -> None: + assert evaluate_cases([]).model_dump() == { + "extraction_precision": 0.0, + "extraction_recall": 0.0, + "top_1_accuracy": 0.0, + "top_k_recall": 0.0, + "auto_precision": 0.0, + "coverage": 0.0, + "ambiguous_accuracy": 0.0, + "sensitive_field_precision": 0.0, + "document_zero_error_rate": 0.0, + } + + +@pytest.mark.parametrize( + "updates", + ( + {"expected_canonical_field_id": "identity." + "x" * 200}, + {"predicted_canonical_field_id": "identity." + "x" * 200}, + {"candidate_ids": ("identity." + "x" * 200,)}, + ), +) +def test_evaluation_bounds_canonical_and_candidate_ids( + updates: dict[str, object], +) -> None: + payload: dict[str, object] = { + "case_id": "bounded-case", + "document_id": "bounded-document", + "expected_status": "MATCHED", + "expected_canonical_field_id": "company.phone", + "expected_sensitive": False, + "predicted_status": "MATCHED", + "predicted_canonical_field_id": "company.phone", + "candidate_ids": ("company.phone",), + } + payload.update(updates) + + with pytest.raises(ValidationError): + EvaluationCase.model_validate(payload) + + +def test_cli_rejects_unbounded_fixture_context_before_mapping( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + cases_path = tmp_path / "cases.jsonl" + cases_path.write_text( + json.dumps( + { + "case_id": "oversized-context", + "document_id": "document-1", + "context": { + "field_id": "field-1", + "label": "x" * 201, + "normalized_label": "x" * 200, + "field_type": "text", + "document_title": "Application", + "section": "Company", + "row_labels": ["Company"], + "nearby_labels": [], + "options": [], + "repeat_index": 0, + "required": True, + "kind": "text_field", + }, + "expected_status": "MATCHED", + "expected_canonical_field_id": "company.name", + } + ) + + "\n", + encoding="utf-8", + ) + output_path = tmp_path / "report.json" + monkeypatch.setattr( + evaluation, + "_run_cases", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("unbounded fixture reached mapping") + ), + ) + + exit_code = evaluation.main( + [ + "--cases", + str(cases_path), + "--catalog", + "unused.yaml", + "--mode", + "rule", + "--output", + str(output_path), + ] + ) + + assert exit_code == 1 + assert not output_path.exists() + + +def test_rule_mode_cli_writes_json_without_model_packages(tmp_path: Path) -> None: + output_path = tmp_path / "baseline.json" + result = subprocess.run( + [ + sys.executable, + "scripts/evaluate_dynamic_mapping.py", + "--cases", + str(CASES_PATH), + "--catalog", + str(CATALOG_PATH), + "--mode", + "rule", + "--output", + str(output_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(output_path.read_text(encoding="utf-8")) + assert report["mode"] == "rule" + assert report["catalog_version"] == "v1" + assert report["metrics"]["auto_precision"] == 1.0 + assert report["metrics"]["coverage"] == 0.4 + assert report["gate"]["passed"] is True + assert "precision=1.000000" in result.stdout + assert "coverage=0.400000" in result.stdout + + +def test_qwen_cli_fails_closed_when_lazy_model_inference_never_succeeds( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class UnavailableSentenceTransformer: + def __init__(self, *_args: object, **_kwargs: object) -> None: + raise RuntimeError("model cache is unavailable") + + monkeypatch.setitem( + sys.modules, + "sentence_transformers", + SimpleNamespace(SentenceTransformer=UnavailableSentenceTransformer), + ) + model_cache = tmp_path / "missing-model-cache" + embedding_path = ( + model_cache + / "qwen3-embedding-0.6b" + / "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3" + ) + reranker_path = ( + model_cache + / "qwen3-reranker-0.6b" + / "e61197ed45024b0ed8a2d74b80b4d909f1255473" + ) + monkeypatch.setenv("FOWOCO_QWEN3_EMBEDDING_PATH", str(embedding_path)) + monkeypatch.setenv("FOWOCO_QWEN3_RERANKER_PATH", str(reranker_path)) + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(model_cache)) + monkeypatch.setenv("FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED", "true") + output_path = tmp_path / "false-green-report.json" + + exit_code = evaluation.main( + [ + "--cases", + str(CASES_PATH), + "--catalog", + str(CATALOG_PATH), + "--mode", + "qwen", + "--output", + str(output_path), + ] + ) + + report = json.loads(output_path.read_text(encoding="utf-8")) + assert exit_code == 2 + assert report["model_execution"] == { + "embedding_success_count": 0, + "reranker_success_count": 0, + "required": True, + "semantic_case_count": 1, + "semantic_case_pass_count": 0, + } + assert report["gate"]["passed"] is False + + +def test_qwen_cli_uses_documented_settings_and_records_fake_backend_execution( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + model_cache = tmp_path / "model-cache" + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(model_cache)) + monkeypatch.setenv("FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED", "true") + monkeypatch.delenv("FOWOCO_QWEN3_EMBEDDING_PATH", raising=False) + monkeypatch.delenv("FOWOCO_QWEN3_RERANKER_PATH", raising=False) + embedding = SemanticEmbeddingBackend() + reranker = SemanticRerankerBackend() + output_path = tmp_path / "qwen-report.json" + + exit_code = evaluation.main( + [ + "--cases", + str(CASES_PATH), + "--catalog", + str(CATALOG_PATH), + "--mode", + "qwen", + "--output", + str(output_path), + ], + embedding_backend=embedding, + reranker_backend=reranker, + ) + + report = json.loads(output_path.read_text(encoding="utf-8")) + assert exit_code == 0 + assert embedding.query_batches > 0 + assert embedding.document_batches > 0 + assert reranker.pair_batches > 0 + assert report["model_execution"] == { + "embedding_success_count": 2, + "reranker_success_count": 2, + "required": True, + "semantic_case_count": 1, + "semantic_case_pass_count": 1, + } + assert report["gate"]["passed"] is True diff --git a/tests/documents/dynamic_automation/test_feedback.py b/tests/documents/dynamic_automation/test_feedback.py new file mode 100644 index 0000000..5ae19e2 --- /dev/null +++ b/tests/documents/dynamic_automation/test_feedback.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from app.documents.dynamic_automation.feedback import ( + JsonlMappingFeedbackStore, + MappingFeedbackRecord, + ReviewerDecision, +) +from app.documents.dynamic_automation.models import ( + CanonicalMappingPlan, + DocumentFieldContext, + FieldMapping, + MappingEvidence, + MappingStatus, + ScoredCandidate, +) + + +def feedback_payload(**updates: object) -> dict[str, object]: + payload: dict[str, object] = { + "schema_version": "v2", + "layout_hash": "a" * 64, + "document_kind": "integrated_application", + "document_version": "v34", + "source_institution": "immigration_office", + "field_context_hash": "b" * 64, + "field_id": "passport-field-1", + "repeat_index": 0, + "label": "Passport number", + "section": "Identity", + "row_labels": ["Identity", "Passport number"], + "nearby_labels": ["Nationality"], + "predicted_status": "MATCHED", + "predicted_canonical_field_id": "identity.passport_number", + "final_canonical_field_id": "identity.passport_number", + "decision": "accepted", + "candidate_scores": [ + { + "canonical_field_id": "identity.passport_number", + "score": 1.0, + "rank": 1, + } + ], + "catalog_version": "v1", + "model_version": "deterministic-rules-v1", + } + payload.update(updates) + return payload + + +def test_feedback_schema_has_no_value_field() -> None: + schema = MappingFeedbackRecord.model_json_schema() + serialized = json.dumps(schema) + + assert "resolved_value" not in serialized + assert "db_value" not in serialized + assert "document_value" not in serialized + + +@pytest.mark.parametrize( + "forbidden_key", + ("resolved_value", "passport", "registration_number", "resident_number"), +) +def test_feedback_rejects_extra_and_sensitive_keys(forbidden_key: str) -> None: + with pytest.raises(ValidationError): + MappingFeedbackRecord.model_validate( + {**feedback_payload(), forbidden_key: "must-never-be-persisted"} + ) + + +def test_sensitive_words_are_allowed_inside_valid_canonical_ids() -> None: + record = MappingFeedbackRecord.model_validate(feedback_payload()) + + assert record.predicted_canonical_field_id == "identity.passport_number" + assert record.final_canonical_field_id == "identity.passport_number" + assert record.candidate_scores[0].canonical_field_id == "identity.passport_number" + + +def test_feedback_bounds_structural_text() -> None: + with pytest.raises(ValidationError): + MappingFeedbackRecord.model_validate(feedback_payload(label="x" * 201)) + + +@pytest.mark.parametrize( + "metadata_field", + ("document_kind", "document_version", "source_institution"), +) +def test_feedback_requires_bounded_group_metadata(metadata_field: str) -> None: + complete = MappingFeedbackRecord.model_validate(feedback_payload()) + assert getattr(complete, metadata_field) + missing = feedback_payload() + missing.pop(metadata_field) + + with pytest.raises(ValidationError): + MappingFeedbackRecord.model_validate(missing) + with pytest.raises(ValidationError): + MappingFeedbackRecord.model_validate( + feedback_payload(**{metadata_field: "x" * 101}) + ) + + +@pytest.mark.parametrize( + "updates", + ( + {"predicted_canonical_field_id": "identity." + "x" * 200}, + {"final_canonical_field_id": "identity." + "x" * 200}, + { + "candidate_scores": [ + { + "canonical_field_id": "identity." + "x" * 200, + "score": 1.0, + "rank": 1, + } + ] + }, + {"catalog_version": "v" + "1" * 200}, + ), +) +def test_feedback_bounds_every_persisted_identifier(updates: dict[str, object]) -> None: + with pytest.raises(ValidationError): + MappingFeedbackRecord.model_validate(feedback_payload(**updates)) + + +def test_feedback_rejects_inconsistent_matched_prediction() -> None: + with pytest.raises(ValidationError): + MappingFeedbackRecord.model_validate( + feedback_payload(predicted_canonical_field_id=None) + ) + + +def test_feedback_builds_deterministic_record_from_mapping_plan() -> None: + context = DocumentFieldContext( + field_id="passport-field-1", + container_id="section0.table0", + label="Passport number", + normalized_label="passportnumber", + field_type="text", + document_title="Employment application", + section="Identity", + row_labels=("Identity", "Passport number"), + nearby_labels=("Nationality",), + options=(), + repeat_index=0, + required=True, + kind="text_field", + ) + plan = CanonicalMappingPlan( + catalog_version="v1", + mappings=( + FieldMapping( + field_id=context.field_id, + repeat_index=0, + status=MappingStatus.MATCHED, + canonical_field_id="identity.passport_number", + candidates=( + ScoredCandidate( + canonical_field_id="identity.passport_number", score=1.0, rank=1 + ), + ), + evidence=MappingEvidence( + reason="exact_alias", + catalog_version="v1", + model_version="deterministic-rules-v1", + ), + ), + ), + ) + + record = MappingFeedbackRecord.from_review( + plan, + context, + layout_hash="a" * 64, + document_kind="integrated_application", + document_version="v34", + source_institution="immigration_office", + decision=ReviewerDecision.ACCEPTED, + final_canonical_field_id="identity.passport_number", + ) + + assert record.field_context_hash == ( + "92f5f396b872309c1cb71d37dede66b6a04128c0b3f6619b0b6f6f04bf0298b4" + ) + assert [candidate.model_dump() for candidate in record.candidate_scores] == [ + { + "canonical_field_id": "identity.passport_number", + "score": 1.0, + "rank": 1, + } + ] + assert record.model_version == "deterministic-rules-v1" + + +def test_feedback_store_appends_one_valid_json_object_per_line(tmp_path: Path) -> None: + path = tmp_path / "mapping-feedback.jsonl" + store = JsonlMappingFeedbackStore(path) + first = MappingFeedbackRecord.model_validate(feedback_payload()) + second = MappingFeedbackRecord.model_validate( + feedback_payload( + field_context_hash="c" * 64, + predicted_status=MappingStatus.AMBIGUOUS, + predicted_canonical_field_id=None, + final_canonical_field_id="worker.legal_name", + decision=ReviewerDecision.CORRECTED, + candidate_scores=[], + ) + ) + + store.append(first) + store.append(second) + + lines = path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["decision"] == "accepted" + assert json.loads(lines[1])["decision"] == "corrected" + assert "must-never-be-persisted" not in path.read_text(encoding="utf-8") + + +def test_feedback_store_accepts_records_only(tmp_path: Path) -> None: + store = JsonlMappingFeedbackStore(tmp_path / "mapping-feedback.jsonl") + + with pytest.raises(TypeError): + store.append(feedback_payload()) # type: ignore[arg-type] + + +def test_feedback_store_revalidates_constructed_records_before_writing(tmp_path: Path) -> None: + path = tmp_path / "mapping-feedback.jsonl" + store = JsonlMappingFeedbackStore(path) + unvalidated = MappingFeedbackRecord.model_construct( + **feedback_payload(label="x" * 201) + ) + + with pytest.raises(ValidationError): + store.append(unvalidated) + + assert not path.exists() diff --git a/tests/documents/dynamic_automation/test_field_context.py b/tests/documents/dynamic_automation/test_field_context.py new file mode 100644 index 0000000..1f3479a --- /dev/null +++ b/tests/documents/dynamic_automation/test_field_context.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import get_args + +import pytest + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.field_context import build_field_contexts +from app.documents.dynamic_automation.models import RegistryFieldType as MappingRegistryFieldType + +HWP_EDITOR_SRC = Path(__file__).parents[3] / "hwp-editor" / "src" +if str(HWP_EDITOR_SRC) not in sys.path: + sys.path.insert(0, str(HWP_EDITOR_SRC)) + +from hwp_mcp.fields import RegistryField # noqa: E402 +from hwp_mcp.fields import RegistryFieldType as McpRegistryFieldType # noqa: E402 + +FIXTURE_DIR = Path(__file__).parents[2] / "fixtures" / "dynamic_automation" +CATALOG_PATH = ( + Path(__file__).parents[3] + / "app/documents/dynamic_automation/resources/canonical_fields.v1.yaml" +) + + +def registry_field( + *, + field_id: str, + target_id: str, + label: str, + row: int, + column: int, + field_type: str = "phone", +) -> dict[str, object]: + """Mirror every field emitted by MCP RegistryField.model_dump().""" + return { + "field_id": field_id, + "target_id": target_id, + "label": label, + "type": field_type, + "category": "step1_application", + "row": row, + "column": column, + "current_text": "", + "required": True, + "options": None, + "kind": "text_field", + "xml_segments": [target_id], + "visual_regions": [], + "constraints": {}, + "disposition": None, + } + + +@pytest.fixture +def registry_fixture() -> list[dict[str, object]]: + return json.loads( + (FIXTURE_DIR / "integrated_application_registry.json").read_text(encoding="utf-8") + ) + + +def test_context_distinguishes_company_phone_from_worker_phone( + registry_fixture: list[dict[str, object]], +) -> None: + contexts = build_field_contexts(registry_fixture, document_title="통합신청서") + + phone = next(item for item in contexts if item.field_id == "workplace-phone") + worker_phone = next(item for item in contexts if item.field_id == "worker-phone") + + assert phone.row_labels == ("현재 근무처", "사업자등록번호", "전화번호:") + assert phone.section == "현재 근무처" + assert worker_phone.row_labels == ("근로자", "전화번호") + assert worker_phone.repeat_index == 1 + + +def test_context_normalizes_and_bounds_untrusted_text() -> None: + contexts = build_field_contexts( + [ + { + "field_id": "field-1", + "target_id": "section0.table0.row1.cell1", + "label": " Company—Phone: ", + "type": "phone", + "kind": "text_field", + "row": 1, + "column": 1, + "required": True, + "options": ["x" * 201], + } + ], + document_title="t" * 201, + ) + + context = contexts[0] + assert context.normalized_label == "companyphone" + assert context.document_title == "t" * 200 + assert context.options == ("x" * 200,) + + +def test_context_accepts_registry_null_options() -> None: + contexts = build_field_contexts( + [ + { + "field_id": "field-1", + "target_id": "section0.table0.row1.cell1", + "label": "Name", + "type": "text", + "kind": "text_field", + "row": 1, + "column": 1, + "required": True, + "options": None, + } + ], + document_title="Application", + ) + + assert contexts[0].options == () + + +def test_equal_coordinates_in_different_tables_are_container_local() -> None: + registry = [ + registry_field( + field_id="company-heading", + target_id="section0.table0.row0.cell0", + label="Company", + row=0, + column=0, + field_type="text", + ), + registry_field( + field_id="company-phone", + target_id="section0.table0.row0.cell1", + label="Phone", + row=0, + column=1, + ), + registry_field( + field_id="company-address", + target_id="section0.table0.row1.cell0", + label="Company address", + row=1, + column=0, + field_type="text", + ), + registry_field( + field_id="worker-heading", + target_id="section0.table1.row0.cell0", + label="Worker", + row=0, + column=0, + field_type="text", + ), + registry_field( + field_id="worker-phone", + target_id="section0.table1.row0.cell1", + label="Phone", + row=0, + column=1, + ), + registry_field( + field_id="worker-nationality", + target_id="section0.table1.row1.cell0", + label="Worker nationality", + row=1, + column=0, + field_type="text", + ), + ] + + contexts = build_field_contexts(registry, document_title="Application") + company = next(item for item in contexts if item.field_id == "company-phone") + worker = next(item for item in contexts if item.field_id == "worker-phone") + + assert company.container_id == "section0.table0" + assert company.row_labels == ("Company", "Phone") + assert company.section == "Company" + assert company.nearby_labels == ("Company address",) + assert company.repeat_index == 0 + assert worker.container_id == "section0.table1" + assert worker.row_labels == ("Worker", "Phone") + assert worker.section == "Worker" + assert worker.nearby_labels == ("Worker nationality",) + assert worker.repeat_index == 0 + + +def test_repeated_worker_and_company_phone_contexts_keep_compatible_candidates() -> None: + registry = [ + registry_field( + field_id="company-phone", + target_id="section0.table0.row0.cell0", + label="Phone", + row=0, + column=0, + ), + registry_field( + field_id="worker-phone", + target_id="section0.table0.row1.cell0", + label="Phone", + row=1, + column=0, + ), + ] + contexts = build_field_contexts(registry, document_title="Application") + catalog = CanonicalCatalog.load(CATALOG_PATH) + + assert contexts[1].repeat_index == 1 + for context in contexts: + candidate_ids = {item.field_id for item in catalog.compatible(context)} + assert {"worker.phone", "company.phone"} <= candidate_ids + + +def test_registry_rejects_duplicate_and_oversized_field_identities() -> None: + duplicate = registry_field( + field_id="duplicate-field", + target_id="section0.table0.row0.cell0", + label="Name", + row=0, + column=0, + field_type="text", + ) + second = { + **duplicate, + "target_id": "section0.table0.row1.cell0", + "row": 1, + "xml_segments": ["section0.table0.row1.cell0"], + } + + with pytest.raises(ValueError, match="duplicate field_id"): + build_field_contexts([duplicate, second], document_title="Application") + + oversized = { + **duplicate, + "field_id": "x" * 201, + } + with pytest.raises(ValueError, match="field_id"): + build_field_contexts([oversized], document_title="Application") + + +def test_actual_mcp_number_fields_keep_identifier_candidates( + registry_fixture: list[dict[str, object]], +) -> None: + contexts = build_field_contexts(registry_fixture, document_title="통합신청서") + catalog = CanonicalCatalog.load(CATALOG_PATH) + business = next( + item for item in contexts if item.field_id == "business-registration-number" + ) + alien = next( + item for item in contexts if item.field_id == "alien-registration-number" + ) + + assert business.field_type == "number" + assert alien.field_type == "number" + assert "company.business_number" in { + item.field_id for item in catalog.compatible(business) + } + assert "identity.alien_registration_number" in { + item.field_id for item in catalog.compatible(alien) + } + + +def test_registry_fixture_and_type_union_match_actual_mcp_contract( + registry_fixture: list[dict[str, object]], +) -> None: + serialized = [ + RegistryField.model_validate(item).model_dump(mode="json") + for item in registry_fixture + ] + + assert serialized == registry_fixture + assert set(get_args(MappingRegistryFieldType)) == set(get_args(McpRegistryFieldType)) diff --git a/tests/documents/dynamic_automation/test_global_validation.py b/tests/documents/dynamic_automation/test_global_validation.py new file mode 100644 index 0000000..3b3e02a --- /dev/null +++ b/tests/documents/dynamic_automation/test_global_validation.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from pathlib import Path + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.global_validation import validate_global_mapping +from app.documents.dynamic_automation.models import ( + CanonicalMappingPlan, + FieldMapping, + MappingEvidence, + MappingStatus, +) + +CATALOG_YAML = """ +version: v1 +fields: + - field_id: worker.legal_name + entity: worker + value_type: string + aliases: [Name] + description: Worker's legal name. + compatible_field_types: [text] + repeatable: false + source: {view: document_worker_view, column: legal_name, scope_keys: [tenant_id, worker_id]} + sensitivity: personal + formatter: person_name + - field_id: worker.dependent_name + entity: worker + value_type: string + aliases: [Dependent name] + description: Repeated dependent name. + compatible_field_types: [text] + repeatable: true + source: {view: document_worker_view, column: dependent_name, scope_keys: [tenant_id, worker_id]} + sensitivity: personal + formatter: person_name + - field_id: company.name + entity: company + value_type: string + aliases: [Company] + description: Company name. + compatible_field_types: [text] + repeatable: false + source: {view: document_company_view, column: name, scope_keys: [tenant_id, company_id]} + sensitivity: business + formatter: string +""" + + +def make_catalog(tmp_path: Path) -> CanonicalCatalog: + path = tmp_path / "catalog.yaml" + path.write_text(CATALOG_YAML, encoding="utf-8") + return CanonicalCatalog.load(path) + + +def matched( + field_id: str, + canonical_field_id: str, + *, + repeat_index: int = 0, + entity_hint: str | None = None, +) -> FieldMapping: + return FieldMapping( + field_id=field_id, + repeat_index=repeat_index, + status=MappingStatus.MATCHED, + canonical_field_id=canonical_field_id, + evidence=MappingEvidence( + reason="decision_gate_passed", + rule="semantic_decision_gate", + embedding_rank=1, + reranker_score=0.96, + score_margin=0.20, + type_compatible=True, + entity_hint=entity_hint, + catalog_version="v1", + model_version="fake-reranker-v1", + ), + ) + + +def test_three_non_repeatable_identity_fields_are_all_downgraded(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mappings = tuple( + matched(f"name-{index}", "worker.legal_name", repeat_index=index) + for index in range(3) + ) + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v1", mappings=mappings), catalog + ) + + assert [item.status for item in result.mappings] == [MappingStatus.AMBIGUOUS] * 3 + assert {item.evidence.reason for item in result.mappings} == { + "duplicate_non_repeatable_canonical_field" + } + + +def test_repeatable_field_allows_distinct_repeat_indexes(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mappings = ( + matched("dependent-1", "worker.dependent_name", repeat_index=0), + matched("dependent-2", "worker.dependent_name", repeat_index=1), + ) + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v1", mappings=mappings), catalog + ) + + assert [item.status for item in result.mappings] == [MappingStatus.MATCHED] * 2 + + +def test_repeatable_field_duplicate_repeat_index_is_downgraded(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mappings = ( + matched("dependent-1", "worker.dependent_name", repeat_index=1), + matched("dependent-2", "worker.dependent_name", repeat_index=1), + ) + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v1", mappings=mappings), catalog + ) + + assert [item.status for item in result.mappings] == [MappingStatus.AMBIGUOUS] * 2 + assert {item.evidence.reason for item in result.mappings} == {"duplicate_repeat_index"} + + +def test_incompatible_entity_role_is_downgraded(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mapping = matched("employer", "company.name", entity_hint="worker") + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v1", mappings=(mapping,)), catalog + ) + + assert result.mappings[0].status is MappingStatus.AMBIGUOUS + assert result.mappings[0].evidence.reason == "incompatible_entity_role" + + +def test_incomplete_matched_evidence_is_downgraded(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mapping = FieldMapping( + field_id="worker-name", + repeat_index=0, + status=MappingStatus.MATCHED, + canonical_field_id="worker.legal_name", + evidence=MappingEvidence(reason="decision_gate_passed", catalog_version="v1"), + ) + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v1", mappings=(mapping,)), catalog + ) + + assert result.mappings[0].status is MappingStatus.AMBIGUOUS + assert result.mappings[0].evidence.reason == "incomplete_mapping_evidence" + + +def test_catalog_version_mismatch_is_downgraded(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + base_mapping = matched("worker-name", "worker.legal_name") + mapping = base_mapping.model_copy( + update={ + "evidence": base_mapping.evidence.model_copy(update={"catalog_version": "v2"}) + } + ) + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v2", mappings=(mapping,)), catalog + ) + + assert result.mappings[0].status is MappingStatus.AMBIGUOUS + assert result.mappings[0].evidence.reason == "catalog_version_mismatch" + + +def test_unknown_canonical_id_is_downgraded(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mapping = matched("rogue", "rogue.value") + + result = validate_global_mapping( + CanonicalMappingPlan(catalog_version="v1", mappings=(mapping,)), catalog + ) + + assert result.mappings[0].status is MappingStatus.AMBIGUOUS + assert result.mappings[0].evidence.reason == "unknown_canonical_field" + + +def test_non_data_and_unmapped_fields_remain_unchanged(tmp_path: Path) -> None: + catalog = make_catalog(tmp_path) + mappings = ( + FieldMapping( + field_id="official", + repeat_index=0, + status=MappingStatus.NON_DATA, + evidence=MappingEvidence(reason="official_region", catalog_version="v1"), + ), + FieldMapping( + field_id="unknown", + repeat_index=0, + status=MappingStatus.UNMAPPED, + evidence=MappingEvidence(reason="no_match", catalog_version="v1"), + ), + ) + + plan = CanonicalMappingPlan(catalog_version="v1", mappings=mappings) + + assert validate_global_mapping(plan, catalog) == plan diff --git a/tests/documents/dynamic_automation/test_mapper.py b/tests/documents/dynamic_automation/test_mapper.py new file mode 100644 index 0000000..03f88c3 --- /dev/null +++ b/tests/documents/dynamic_automation/test_mapper.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from fakes import FakeCandidateReranker, FakeCandidateRetriever, scored +from pydantic import ValidationError + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.mapper import HybridFieldMapper, MappingThresholds +from app.documents.dynamic_automation.models import ( + DocumentFieldContext, + FieldMapping, + MappingEvidence, + MappingStatus, + ScoredCandidate, +) + +CATALOG_PATH = ( + Path(__file__).parents[3] + / "app" + / "documents" + / "dynamic_automation" + / "resources" + / "canonical_fields.v1.yaml" +) + + +@pytest.fixture +def catalog() -> CanonicalCatalog: + return CanonicalCatalog.load(CATALOG_PATH) + + +def make_context( + *, + field_id: str = "field-1", + label: str = "연락처", + field_type: str = "phone", + section: str = "현재 근무처", + row_labels: tuple[str, ...] = ("현재 근무처", "연락처"), + repeat_index: int = 0, + kind: str = "text_field", +) -> DocumentFieldContext: + return DocumentFieldContext( + field_id=field_id, + label=label, + normalized_label=label, + field_type=field_type, + document_title="통합신청서", + section=section, + row_labels=row_labels, + nearby_labels=(), + options=(), + repeat_index=repeat_index, + required=True, + kind=kind, + ) + + +def make_mapper( + catalog: CanonicalCatalog, + *, + retrieved: tuple[ScoredCandidate, ...] | None = (), + reranked: tuple[ScoredCandidate, ...] | None = (), + retriever_error: Exception | None = None, + reranker_error: Exception | None = None, + min_score: float = 0.90, + min_margin: float = 0.10, +) -> HybridFieldMapper: + return HybridFieldMapper( + catalog=catalog, + retriever=FakeCandidateRetriever(results=retrieved, error=retriever_error), + reranker=FakeCandidateReranker(results=reranked, error=reranker_error), + thresholds=MappingThresholds( + min_reranker_score=min_score, + min_margin=min_margin, + ), + top_k=5, + ) + + +def test_field_mapping_records_repeat_index() -> None: + mapping = FieldMapping( + field_id="worker-name-1", + repeat_index=1, + status=MappingStatus.UNMAPPED, + evidence=MappingEvidence(reason="no_match", catalog_version="v1"), + ) + + assert mapping.repeat_index == 1 + + +def test_field_mapping_rejects_negative_repeat_index() -> None: + with pytest.raises(ValidationError): + FieldMapping( + field_id="worker-name-1", + repeat_index=-1, + status=MappingStatus.UNMAPPED, + evidence=MappingEvidence(reason="no_match", catalog_version="v1"), + ) + + +def test_mapper_requires_absolute_score_and_margin(catalog: CanonicalCatalog) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.94), ("worker.phone", 0.92)), + reranked=scored(("company.phone", 0.91), ("worker.phone", 0.88)), + min_score=0.90, + min_margin=0.10, + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "insufficient_margin" + assert result.evidence.score_margin == pytest.approx(0.03) + + +def test_mapper_rejects_top_candidate_below_absolute_threshold( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.95), ("worker.phone", 0.80)), + reranked=scored(("company.phone", 0.89), ("worker.phone", 0.60)), + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "insufficient_score" + + +def test_mapper_requires_a_runner_up_for_margin_evidence(catalog: CanonicalCatalog) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.98)), + reranked=scored(("company.phone", 0.96)), + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "insufficient_margin_evidence" + assert result.evidence.score_margin is None + + +def test_reranker_failure_does_not_accept_embedding_top_one( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.99), ("worker.phone", 0.70)), + reranker_error=RuntimeError("offline"), + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "reranker_unavailable" + assert result.candidates[0].canonical_field_id == "company.phone" + + +def test_none_reranker_result_is_ambiguous(catalog: CanonicalCatalog) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.95), ("worker.phone", 0.80)), + reranked=None, + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "invalid_reranker_evidence" + assert result.candidates == () + + +def test_retriever_failure_reduces_coverage_without_lowering_thresholds( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper(catalog, retriever_error=RuntimeError("offline")) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "retriever_unavailable" + + +def test_none_retriever_result_is_ambiguous(catalog: CanonicalCatalog) -> None: + mapper = make_mapper(catalog, retrieved=None) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "invalid_retrieval_evidence" + assert result.candidates == () + + +def test_unique_exact_alias_is_safe_without_model_services( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retriever_error=RuntimeError("offline"), + reranker_error=RuntimeError("offline"), + ) + + result = mapper.map( + (make_context(label="사업장 전화번호", row_labels=("사업장", "전화번호")),) + ).mappings[0] + + assert result.status is MappingStatus.MATCHED + assert result.canonical_field_id == "company.phone" + assert result.evidence.rule == "exact_alias" + assert result.evidence.type_compatible is True + assert result.evidence.catalog_version == "v1" + + +def test_exact_alias_requires_a_compatible_structural_entity_hint( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retriever_error=RuntimeError("offline"), + reranker_error=RuntimeError("offline"), + ) + + result = mapper.map( + (make_context(label="사업장 전화번호", section="", row_labels=()),) + ).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "retriever_unavailable" + assert result.evidence.entity_hint is None + + +def test_mapper_records_complete_semantic_decision_evidence( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.95), ("worker.phone", 0.82)), + reranked=scored(("company.phone", 0.96), ("worker.phone", 0.61)), + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.MATCHED + assert result.canonical_field_id == "company.phone" + assert result.repeat_index == 0 + assert result.evidence.rule == "semantic_decision_gate" + assert result.evidence.embedding_rank == 1 + assert result.evidence.reranker_score == 0.96 + assert result.evidence.score_margin == pytest.approx(0.35) + assert result.evidence.type_compatible is True + assert result.evidence.entity_hint == "company" + assert result.evidence.catalog_version == "v1" + assert result.evidence.model_version == "fake-reranker-v1" + + +def test_mapper_rejects_candidates_outside_the_compatible_allowlist( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("worker.date_of_birth", 0.99), ("company.phone", 0.95)), + reranked=scored(("worker.date_of_birth", 0.99), ("company.phone", 0.95)), + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "invalid_retrieval_evidence" + assert result.evidence.type_compatible is False + + +def test_retriever_result_exceeding_top_k_is_ambiguous(catalog: CanonicalCatalog) -> None: + retrieved = scored(("company.phone", 0.95), ("worker.phone", 0.80)) + mapper = HybridFieldMapper( + catalog=catalog, + retriever=FakeCandidateRetriever(results=retrieved, enforce_top_k=False), + reranker=FakeCandidateReranker(results=retrieved), + thresholds=MappingThresholds(min_reranker_score=0.90, min_margin=0.10), + top_k=1, + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "invalid_retrieval_evidence" + + +def test_malformed_ranking_preserves_truthful_type_compatibility( + catalog: CanonicalCatalog, +) -> None: + mapper = make_mapper( + catalog, + retrieved=scored(("company.phone", 0.95), ("company.phone", 0.80)), + ) + + result = mapper.map((make_context(),)).mappings[0] + + assert result.status is MappingStatus.AMBIGUOUS + assert result.evidence.reason == "invalid_retrieval_evidence" + assert result.evidence.type_compatible is True + + +def test_non_data_fields_bypass_models(catalog: CanonicalCatalog) -> None: + mapper = make_mapper( + catalog, + retriever_error=AssertionError("retriever must not run"), + reranker_error=AssertionError("reranker must not run"), + ) + + result = mapper.map((make_context(label="확인검토", field_type="text"),)).mappings[0] + + assert result.status is MappingStatus.NON_DATA + assert result.evidence.reason == "process_flow_label" + + +def test_no_compatible_candidates_is_unmapped(catalog: CanonicalCatalog) -> None: + mapper = make_mapper(catalog) + + result = mapper.map((make_context(field_type="unsupported"),)).mappings[0] + + assert result.status is MappingStatus.UNMAPPED + assert result.evidence.reason == "no_compatible_candidates" diff --git a/tests/documents/dynamic_automation/test_mapping_config.py b/tests/documents/dynamic_automation/test_mapping_config.py new file mode 100644 index 0000000..cbba92e --- /dev/null +++ b/tests/documents/dynamic_automation/test_mapping_config.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from app.core.config import Settings +from app.documents.dynamic_automation.config import DynamicAutomationSettings + + +def test_invalid_dynamic_environment_cannot_break_server_startup() -> None: + project_root = Path(__file__).parents[3] + environment = { + **os.environ, + "FOWOCO_DYNAMIC_AUTOMATION_MIN_MARGIN": "not-a-number", + "FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH": "outside-cache/model", + } + + completed = subprocess.run( + [sys.executable, "-c", "from app.main import create_app; create_app()"], + cwd=project_root, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_shared_settings_do_not_expose_dynamic_automation_fields() -> None: + settings = Settings(_env_file=None) + + assert not hasattr(settings, "dynamic_automation_mapping_enabled") + assert not hasattr(settings, "dynamic_automation_embedding_model_path") + + +def test_core_config_import_does_not_require_document_automation_extras() -> None: + project_root = Path(__file__).parents[3] + + completed = subprocess.run( + [ + sys.executable, + "-c", + "import sys; sys.modules['yaml'] = None; import app.core.config", + ], + cwd=project_root, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_bge_downloader_import_does_not_require_document_automation_extras() -> None: + project_root = Path(__file__).parents[3] + + completed = subprocess.run( + [ + sys.executable, + "-c", + "import sys; sys.modules['yaml'] = None; import scripts.download_language_models", + ], + cwd=project_root, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_dynamic_mapping_is_disabled_and_uses_pinned_cache_paths_by_default( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(tmp_path)) + monkeypatch.delenv("FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED", raising=False) + monkeypatch.delenv("FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH", raising=False) + monkeypatch.delenv("FOWOCO_DYNAMIC_AUTOMATION_RERANKER_MODEL_PATH", raising=False) + + settings = DynamicAutomationSettings(_env_file=None) + + assert settings.dynamic_automation_mapping_enabled is False + assert settings.dynamic_automation_embedding_model_path == ( + tmp_path + / "qwen3-embedding-0.6b" + / "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3" + ) + assert settings.dynamic_automation_reranker_model_path == ( + tmp_path + / "qwen3-reranker-0.6b" + / "e61197ed45024b0ed8a2d74b80b4d909f1255473" + ) + + +def test_dynamic_mapping_settings_read_explicit_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + model_cache_dir = tmp_path / "model-cache" + embedding_path = ( + model_cache_dir + / "custom" + / "qwen3-embedding-0.6b" + / "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3" + ) + reranker_path = ( + model_cache_dir + / "custom" + / "qwen3-reranker-0.6b" + / "e61197ed45024b0ed8a2d74b80b4d909f1255473" + ) + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(model_cache_dir)) + monkeypatch.setenv("FOWOCO_DYNAMIC_AUTOMATION_MAPPING_ENABLED", "true") + monkeypatch.setenv( + "FOWOCO_DYNAMIC_AUTOMATION_EMBEDDING_MODEL_PATH", str(embedding_path) + ) + monkeypatch.setenv("FOWOCO_DYNAMIC_AUTOMATION_RERANKER_MODEL_PATH", str(reranker_path)) + monkeypatch.setenv("FOWOCO_DYNAMIC_AUTOMATION_MIN_RERANKER_SCORE", "0.93") + monkeypatch.setenv("FOWOCO_DYNAMIC_AUTOMATION_MIN_MARGIN", "0.12") + + settings = DynamicAutomationSettings(_env_file=None) + + assert settings.dynamic_automation_mapping_enabled is True + assert settings.dynamic_automation_embedding_model_path == embedding_path + assert settings.dynamic_automation_reranker_model_path == reranker_path + assert settings.dynamic_automation_min_reranker_score == pytest.approx(0.93) + assert settings.dynamic_automation_min_margin == pytest.approx(0.12) + + +def test_dynamic_mapping_rejects_model_path_outside_managed_cache( + tmp_path: Path, +) -> None: + model_cache_dir = tmp_path / "managed-cache" + outside_path = ( + tmp_path + / "outside-cache" + / "qwen3-embedding-0.6b" + / "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3" + ) + + with pytest.raises(ValidationError, match="must be below model_cache_dir"): + DynamicAutomationSettings( + _env_file=None, + model_cache_dir=model_cache_dir, + dynamic_automation_embedding_model_path=outside_path, + ) + + +def test_dynamic_mapping_rejects_unpinned_revision_inside_managed_cache( + tmp_path: Path, +) -> None: + model_cache_dir = tmp_path / "managed-cache" + unpinned_path = ( + model_cache_dir / "qwen3-reranker-0.6b" / "not-the-pinned-revision" + ) + + with pytest.raises(ValidationError, match="must end in the pinned revision directory"): + DynamicAutomationSettings( + _env_file=None, + model_cache_dir=model_cache_dir, + dynamic_automation_reranker_model_path=unpinned_path, + ) + + +@pytest.mark.parametrize( + ("setting", "value"), + [ + ("dynamic_automation_min_reranker_score", 1.01), + ("dynamic_automation_min_margin", -0.01), + ], +) +def test_dynamic_mapping_thresholds_must_be_probabilities( + setting: str, value: float +) -> None: + with pytest.raises(ValidationError): + DynamicAutomationSettings(_env_file=None, **{setting: value}) + + +def test_qwen_download_specs_are_opt_in_and_pinned(tmp_path: Path) -> None: + from app.documents.dynamic_automation.qwen import ( + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_REPO, + QWEN3_RERANKER_REVISION, + ) + from scripts.download_language_models import ( + DOCUMENT_AUTOMATION_MODEL_SPECS, + verify_model_cache, + ) + + assert DOCUMENT_AUTOMATION_MODEL_SPECS == [ + { + "name": "qwen3-embedding-0.6b", + "repo": QWEN3_EMBEDDING_REPO, + "revision": QWEN3_EMBEDDING_REVISION, + }, + { + "name": "qwen3-reranker-0.6b", + "repo": QWEN3_RERANKER_REPO, + "revision": QWEN3_RERANKER_REVISION, + }, + ] + + assert "qwen3-embedding-0.6b" not in verify_model_cache(tmp_path) + missing = verify_model_cache(tmp_path, include_document_automation=True) + + assert "qwen3-embedding-0.6b" in missing + assert "qwen3-reranker-0.6b" in missing + + (tmp_path / "qwen3-embedding-0.6b" / QWEN3_EMBEDDING_REVISION).mkdir( + parents=True + ) + ( + tmp_path + / "qwen3-embedding-0.6b" + / QWEN3_EMBEDDING_REVISION + / "config.json" + ).write_text("{}") + (tmp_path / "qwen3-reranker-0.6b" / QWEN3_RERANKER_REVISION).mkdir( + parents=True + ) + ( + tmp_path + / "qwen3-reranker-0.6b" + / QWEN3_RERANKER_REVISION + / "config.json" + ).write_text("{}") + + assert not { + "qwen3-embedding-0.6b", + "qwen3-reranker-0.6b", + }.intersection(verify_model_cache(tmp_path, include_document_automation=True)) + + +def test_document_automation_extra_pins_local_model_runtime_dependencies() -> None: + project_root = Path(__file__).parents[3] + configuration = tomllib.loads((project_root / "pyproject.toml").read_text(encoding="utf-8")) + + dependencies = configuration["project"]["optional-dependencies"][ + "document-automation" + ] + + assert "sentence-transformers>=5,<6" in dependencies + assert "transformers>=4.51,<5" in dependencies + assert "torch>=2.2,<3" in dependencies diff --git a/tests/documents/dynamic_automation/test_model_promotion.py b/tests/documents/dynamic_automation/test_model_promotion.py new file mode 100644 index 0000000..86607a9 --- /dev/null +++ b/tests/documents/dynamic_automation/test_model_promotion.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from app.documents.dynamic_automation.qwen import ( + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, +) +from app.documents.dynamic_automation.training import ( + EVALUATION_CODE_VERSION, + TRAINING_CODE_VERSION, + EvaluationMetricsEvidence, + HeldOutEvaluationReport, + ModelManifest, + UnseenFieldEvidence, + compare_manifests, + held_out_evaluation_report_bytes, + held_out_evaluation_report_sha256, +) +from scripts import compare_dynamic_mapping_models as comparison + +ARTIFACT_BYTES = b"deterministic test model artifact\n" +ARTIFACT_SHA256 = hashlib.sha256(ARTIFACT_BYTES).hexdigest() +_REPORTS: dict[str, HeldOutEvaluationReport] = {} + + +def manifest( + *, + auto_precision: float = 0.995, + sensitive_precision: float = 0.996, + coverage: float = 0.80, + ece: float = 0.03, + p95_ms: float = 200, + unseen_field: str | None = "worker.email", + unseen_retrieved: bool = True, + catalog_field_ids: tuple[str, ...] = ( + "company.phone", + "worker.email", + "worker.phone", + ), + dataset_sha256: str = "b" * 64, +) -> ModelManifest: + unseen_id = unseen_field or "worker.email" + candidate_ids = (unseen_id,) if unseen_retrieved and unseen_field is not None else () + report = HeldOutEvaluationReport( + schema_version="dynamic-mapping-held-out-v2", + evaluation_code_version=EVALUATION_CODE_VERSION, + model_artifact_sha256=ARTIFACT_SHA256, + dataset_sha256=dataset_sha256, + catalog_sha256="c" * 64, + catalog_version="v1", + sample_count=10, + cohort_count=5, + model_execution_count=11, + metrics=EvaluationMetricsEvidence( + auto_precision=auto_precision, + sensitive_precision=sensitive_precision, + coverage=coverage, + expected_calibration_error=ece, + p95_latency_ms=p95_ms, + ), + unseen_field_evidence=UnseenFieldEvidence( + case_id=f"generated-unseen:{unseen_id}", + canonical_field_id=unseen_id, + query_sha256="d" * 64, + candidate_ids=candidate_ids, + retrieved_rank=1 if candidate_ids else None, + ), + ) + report_sha256 = held_out_evaluation_report_sha256(report) + _REPORTS[report_sha256] = report + return ModelManifest( + schema_version="dynamic-mapping-model-manifest-v2", + model_kind="domain_bi_encoder", + base_model_repo=QWEN3_EMBEDDING_REPO, + base_model_revision=QWEN3_EMBEDDING_REVISION, + dataset_sha256=dataset_sha256, + catalog_sha256="c" * 64, + model_artifact_sha256=ARTIFACT_SHA256, + evaluation_report_sha256=report_sha256, + catalog_version="v1", + training_code_version=TRAINING_CODE_VERSION, + evaluation_code_version=EVALUATION_CODE_VERSION, + training_sample_count=20, + evaluation_sample_count=10, + training_cohort_count=8, + evaluation_cohort_count=5, + auto_precision=auto_precision, + sensitive_precision=sensitive_precision, + coverage=coverage, + expected_calibration_error=ece, + p95_latency_ms=p95_ms, + training_canonical_field_ids=("worker.phone",), + catalog_field_ids=catalog_field_ids, + ) + + +def baseline_manifest(**updates: float) -> ModelManifest: + return manifest(**updates).model_copy(update={"model_kind": "qwen_baseline"}) + + +def compare_for_test( + *, baseline: ModelManifest, candidate: ModelManifest +): + return compare_manifests( + baseline=baseline, + candidate=candidate, + baseline_report=_REPORTS[baseline.evaluation_report_sha256], + candidate_report=_REPORTS[candidate.evaluation_report_sha256], + baseline_artifact_sha256=ARTIFACT_SHA256, + candidate_artifact_sha256=ARTIFACT_SHA256, + baseline_report_sha256=baseline.evaluation_report_sha256, + candidate_report_sha256=candidate.evaluation_report_sha256, + ) + + +def write_evidence_files( + directory: Path, name: str, model_manifest: ModelManifest +) -> tuple[Path, Path, Path]: + manifest_path = directory / f"{name}-manifest.json" + report_path = directory / f"{name}-report.json" + artifact_path = directory / f"{name}-artifact.json" + manifest_path.write_text(model_manifest.model_dump_json(indent=2), encoding="utf-8") + report_path.write_bytes( + held_out_evaluation_report_bytes( + _REPORTS[model_manifest.evaluation_report_sha256] + ) + ) + artifact_path.write_bytes(ARTIFACT_BYTES) + return manifest_path, report_path, artifact_path + + +def test_model_is_not_promoted_when_precision_or_calibration_regresses() -> None: + decision = compare_for_test( + baseline=baseline_manifest(auto_precision=0.995, ece=0.03, p95_ms=200), + candidate=manifest(auto_precision=0.990, ece=0.04, p95_ms=120), + ) + + assert decision.promote is False + assert "auto_precision" in decision.reasons + assert "expected_calibration_error" in decision.reasons + + +def test_model_is_not_promoted_when_sensitive_precision_regresses() -> None: + decision = compare_for_test( + baseline=baseline_manifest(sensitive_precision=0.997), + candidate=manifest(sensitive_precision=0.996, coverage=0.81), + ) + + assert decision.promote is False + assert "sensitive_precision" in decision.reasons + + +def test_model_is_not_promoted_without_coverage_or_latency_improvement() -> None: + decision = compare_for_test( + baseline=baseline_manifest(coverage=0.80, p95_ms=200), + candidate=manifest(coverage=0.80, p95_ms=200), + ) + + assert decision.promote is False + assert "coverage_or_p95_latency_ms" in decision.reasons + + +def test_model_is_not_promoted_for_catastrophic_coverage_loss_with_latency_gain() -> None: + decision = compare_for_test( + baseline=baseline_manifest(coverage=0.80, p95_ms=200), + candidate=manifest(coverage=0.01, p95_ms=199), + ) + + assert decision.promote is False + assert "coverage" in decision.reasons + + +def test_model_is_not_promoted_for_latency_regression_with_coverage_gain() -> None: + decision = compare_for_test( + baseline=baseline_manifest(coverage=0.80, p95_ms=200), + candidate=manifest(coverage=0.81, p95_ms=999), + ) + + assert decision.promote is False + assert "p95_latency_ms" in decision.reasons + + +def test_model_is_not_promoted_without_unseen_catalog_retrieval() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81, unseen_field=None), + ) + + assert decision.promote is False + assert "unseen_catalog_retrieval" in decision.reasons + + +def test_model_is_not_promoted_for_fabricated_unseen_catalog_id() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81, unseen_field="fabricated.field"), + ) + + assert decision.promote is False + assert "unseen_catalog_retrieval" in decision.reasons + + +def test_model_is_not_promoted_when_unseen_retrieval_evidence_is_false() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81, unseen_retrieved=False), + ) + + assert decision.promote is False + assert "unseen_catalog_retrieval" in decision.reasons + + +def test_model_is_not_promoted_when_unseen_field_was_a_training_label() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81, unseen_field="worker.phone"), + ) + + assert decision.promote is False + assert "unseen_catalog_retrieval" in decision.reasons + + +def test_model_is_promoted_only_when_every_gate_passes() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81), + ) + + assert decision.promote is True + assert decision.reasons == () + + +def test_model_is_not_promoted_against_an_unpinned_qwen_manifest() -> None: + decision = compare_for_test( + baseline=baseline_manifest().model_copy( + update={"base_model_revision": "unreviewed"} + ), + candidate=manifest(coverage=0.81), + ) + + assert decision.promote is False + assert "base_model_manifest" in decision.reasons + + +@pytest.mark.parametrize("catalog_sha256", (None, "0" * 64)) +def test_model_manifest_requires_a_nonzero_catalog_hash( + catalog_sha256: str | None, +) -> None: + payload = manifest().model_dump(mode="json") + if catalog_sha256 is None: + payload.pop("catalog_sha256") + else: + payload["catalog_sha256"] = catalog_sha256 + + with pytest.raises(ValidationError): + ModelManifest.model_validate(payload) + + +def test_model_is_not_promoted_when_catalog_hashes_differ() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81).model_copy( + update={"catalog_sha256": "d" * 64} + ), + ) + + assert decision.promote is False + assert "catalog_sha256" in decision.reasons + + +def test_model_is_not_promoted_when_held_out_datasets_differ() -> None: + decision = compare_for_test( + baseline=baseline_manifest(), + candidate=manifest(coverage=0.81, dataset_sha256="e" * 64), + ) + + assert decision.promote is False + assert "dataset_sha256" in decision.reasons + + +def test_model_manifest_rejects_disconnected_boolean_unseen_evidence() -> None: + payload = manifest().model_dump(mode="json") + payload["unseen_catalog_retrieved"] = True + + with pytest.raises(ValidationError): + ModelManifest.model_validate(payload) + + +def test_comparison_rejects_report_hash_not_bound_to_manifest() -> None: + baseline = baseline_manifest() + candidate = manifest(coverage=0.81) + candidate_report = _REPORTS[candidate.evaluation_report_sha256] + + decision = compare_manifests( + baseline=baseline, + candidate=candidate, + baseline_report=_REPORTS[baseline.evaluation_report_sha256], + candidate_report=candidate_report, + baseline_artifact_sha256=ARTIFACT_SHA256, + candidate_artifact_sha256=ARTIFACT_SHA256, + baseline_report_sha256=baseline.evaluation_report_sha256, + candidate_report_sha256="f" * 64, + ) + + assert decision.promote is False + assert "candidate_evaluation_evidence" in decision.reasons + + +def test_comparison_fails_closed_without_exact_evidence_byte_hashes() -> None: + baseline = baseline_manifest() + candidate = manifest(coverage=0.81) + + decision = compare_manifests( + baseline=baseline, + candidate=candidate, + baseline_report=_REPORTS[baseline.evaluation_report_sha256], + candidate_report=_REPORTS[candidate.evaluation_report_sha256], + ) + + assert decision.promote is False + assert decision.reasons == ( + "baseline_evaluation_evidence", + "candidate_evaluation_evidence", + ) + + +def test_comparison_cli_writes_deterministic_fail_closed_report(tmp_path: Path) -> None: + output_path = tmp_path / "decision.json" + baseline_path, baseline_report, baseline_artifact = write_evidence_files( + tmp_path, "baseline", baseline_manifest() + ) + candidate_path, candidate_report, candidate_artifact = write_evidence_files( + tmp_path, + "candidate", + manifest(auto_precision=0.99, coverage=0.82), + ) + + exit_code = comparison.main( + [ + "--baseline", + str(baseline_path), + "--baseline-artifact", + str(baseline_artifact), + "--baseline-report", + str(baseline_report), + "--candidate", + str(candidate_path), + "--candidate-artifact", + str(candidate_artifact), + "--candidate-report", + str(candidate_report), + "--output", + str(output_path), + ] + ) + + assert exit_code == 2 + assert json.loads(output_path.read_text(encoding="utf-8")) == { + "promote": False, + "reasons": ["auto_precision"], + } + + +def test_comparison_script_runs_from_project_root(tmp_path: Path) -> None: + root = Path(__file__).parents[3] + output_path = tmp_path / "decision.json" + baseline_path, baseline_report, baseline_artifact = write_evidence_files( + tmp_path, "baseline", baseline_manifest() + ) + candidate_path, candidate_report, candidate_artifact = write_evidence_files( + tmp_path, "candidate", manifest(coverage=0.81) + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/compare_dynamic_mapping_models.py", + "--baseline", + str(baseline_path), + "--baseline-artifact", + str(baseline_artifact), + "--baseline-report", + str(baseline_report), + "--candidate", + str(candidate_path), + "--candidate-artifact", + str(candidate_artifact), + "--candidate-report", + str(candidate_report), + "--output", + str(output_path), + ], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + assert json.loads(output_path.read_text(encoding="utf-8")) == { + "promote": True, + "reasons": [], + } diff --git a/tests/documents/dynamic_automation/test_qwen_adapters.py b/tests/documents/dynamic_automation/test_qwen_adapters.py new file mode 100644 index 0000000..9c4f3f4 --- /dev/null +++ b/tests/documents/dynamic_automation/test_qwen_adapters.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import sys +from collections.abc import Sequence +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from app.documents.dynamic_automation.models import ( + CanonicalFieldDefinition, + CanonicalSource, + DocumentFieldContext, + ScoredCandidate, +) +from app.documents.dynamic_automation.qwen import ( + QWEN3_EMBEDDING_REPO, + QWEN3_EMBEDDING_REVISION, + QWEN3_RERANKER_REPO, + QWEN3_RERANKER_REVISION, + LocalQwen3RerankerBackend, + LocalSentenceTransformerBackend, + Qwen3CandidateReranker, + Qwen3EmbeddingRetriever, +) + +COMPANY_PHONE_CONTEXT = DocumentFieldContext( + field_id="field-1", + label="회사 연락처", + normalized_label="회사 연락처", + field_type="phone", + document_title="통합신청서", + section="현재 근무처", + row_labels=("현재 근무처", "연락처"), + nearby_labels=("회사명",), + options=(), + repeat_index=0, + required=True, + kind="text_field", +) + + +def definition( + field_id: str, + *, + entity: str, + description: str, +) -> CanonicalFieldDefinition: + return CanonicalFieldDefinition( + field_id=field_id, + entity=entity, + value_type="phone", + aliases=(field_id.replace(".", " "),), + description=description, + compatible_field_types=("phone", "text"), + source=CanonicalSource( + view=f"document_{entity}_view", + column="phone", + scope_keys=("tenant_id", "company_id") + if entity == "company" + else ("tenant_id", "worker_id"), + ), + sensitivity="business" if entity == "company" else "personal", + formatter="phone", + ) + + +CANDIDATE_DEFINITIONS = ( + definition( + "worker.phone", + entity="worker", + description="Worker's phone number.", + ), + definition( + "company.phone", + entity="company", + description="Company's contact telephone number.", + ), +) +CANDIDATES = ( + ScoredCandidate(canonical_field_id="worker.phone", score=0.8, rank=1), + ScoredCandidate(canonical_field_id="company.phone", score=0.7, rank=2), +) + + +def resolve_definition(field_id: str) -> CanonicalFieldDefinition: + try: + return next(item for item in CANDIDATE_DEFINITIONS if item.field_id == field_id) + except StopIteration as error: + raise KeyError(field_id) from error + + +class RecordingEmbeddingBackend: + def __init__(self) -> None: + self.queries: list[str] = [] + self.documents: list[str] = [] + self.calls: list[tuple[int, int]] = [] + + def encode_queries( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + self.queries.extend(texts) + self.calls.append((max_length, batch_size)) + return ((1.0, 0.0),) + + def encode_documents( + self, + texts: Sequence[str], + *, + max_length: int, + batch_size: int, + ) -> tuple[tuple[float, ...], ...]: + self.documents.extend(texts) + self.calls.append((max_length, batch_size)) + return ((0.0, 1.0), (1.0, 0.0)) + + +class FakeLogitBackend: + def __init__(self, scores: tuple[float, ...]) -> None: + self.scores = scores + self.pairs: list[tuple[str, str]] = [] + self.calls: list[tuple[int, int]] = [] + + def score_pairs( + self, + pairs: Sequence[tuple[str, str]], + *, + max_length: int, + batch_size: int, + ) -> tuple[float, ...]: + self.pairs.extend(pairs) + self.calls.append((max_length, batch_size)) + return self.scores + + +def test_reranker_backend_receives_full_resolved_canonical_definitions() -> None: + backend = FakeLogitBackend(scores=(0.8, 0.2)) + reranker = Qwen3CandidateReranker( + backend=backend, + definition_resolver=resolve_definition, + ) + + reranker.rerank(COMPANY_PHONE_CONTEXT, CANDIDATES) + + worker_definition = backend.pairs[0][1] + assert "canonical field: worker.phone" in worker_definition + assert "entity: worker" in worker_definition + assert "value type: phone" in worker_definition + assert "aliases: worker phone" in worker_definition + assert "description: Worker's phone number." in worker_definition + + +def test_reranker_rejects_unknown_candidate_before_backend_execution() -> None: + backend = FakeLogitBackend(scores=(0.5,)) + reranker = Qwen3CandidateReranker( + backend=backend, + definition_resolver=resolve_definition, + ) + unknown = ( + ScoredCandidate(canonical_field_id="unknown.field", score=0.5, rank=1), + ) + + with pytest.raises(ValueError, match="unknown canonical candidate"): + reranker.rerank(COMPANY_PHONE_CONTEXT, unknown) + + assert backend.pairs == [] + + +def test_embedding_query_includes_instruction_and_structural_context() -> None: + backend = RecordingEmbeddingBackend() + retriever = Qwen3EmbeddingRetriever(backend=backend) + + retriever.retrieve(COMPANY_PHONE_CONTEXT, CANDIDATE_DEFINITIONS, top_k=2) + + assert "회사 연락처 canonical field를 찾으세요" in backend.queries[0] + assert "현재 근무처" in backend.queries[0] + assert "통합신청서" in backend.queries[0] + + +def test_embedding_batches_candidates_and_returns_bounded_ranked_scores() -> None: + backend = RecordingEmbeddingBackend() + retriever = Qwen3EmbeddingRetriever(backend=backend, max_length=512, batch_size=8) + + ranked = retriever.retrieve(COMPANY_PHONE_CONTEXT, CANDIDATE_DEFINITIONS, top_k=1) + + assert len(backend.documents) == 2 + assert "company.phone" in backend.documents[1] + assert backend.calls == [(512, 8), (512, 8)] + assert [(item.canonical_field_id, item.rank) for item in ranked] == [ + ("company.phone", 1) + ] + assert ranked[0].score == pytest.approx(1.0) + + +def test_reranker_uses_yes_no_probability_and_returns_zero_to_one() -> None: + backend = FakeLogitBackend(scores=(0.8, 0.2)) + reranker = Qwen3CandidateReranker( + backend=backend, definition_resolver=resolve_definition + ) + + ranked = reranker.rerank(COMPANY_PHONE_CONTEXT, CANDIDATES) + + assert ranked[0].canonical_field_id == "worker.phone" + assert ranked[0].score == pytest.approx(0.8) + assert ranked[1].score == pytest.approx(0.2) + assert all(0 <= item.score <= 1 for item in ranked) + + +def test_reranker_batches_structural_query_and_definitions_at_512_tokens() -> None: + backend = FakeLogitBackend(scores=(0.2, 0.9)) + reranker = Qwen3CandidateReranker( + backend=backend, + definition_resolver=resolve_definition, + max_length=512, + batch_size=4, + ) + + ranked = reranker.rerank(COMPANY_PHONE_CONTEXT, CANDIDATES) + + assert len(backend.pairs) == 2 + assert "현재 근무처" in backend.pairs[0][0] + assert "canonical field: company.phone" in backend.pairs[1][1] + assert backend.calls == [(512, 4)] + assert [item.canonical_field_id for item in ranked] == [ + "company.phone", + "worker.phone", + ] + assert [item.rank for item in ranked] == [1, 2] + + +def test_adapters_report_pinned_model_versions() -> None: + assert Qwen3EmbeddingRetriever(backend=RecordingEmbeddingBackend()).model_version == ( + f"{QWEN3_EMBEDDING_REPO}@{QWEN3_EMBEDDING_REVISION}" + ) + assert Qwen3CandidateReranker(backend=FakeLogitBackend(())).model_version == ( + f"{QWEN3_RERANKER_REPO}@{QWEN3_RERANKER_REVISION}" + ) + + +@pytest.mark.parametrize( + ("adapter_type", "cache_name", "revision"), + [ + ( + Qwen3EmbeddingRetriever, + "qwen3-embedding-0.6b", + QWEN3_EMBEDDING_REVISION, + ), + ( + Qwen3CandidateReranker, + "qwen3-reranker-0.6b", + QWEN3_RERANKER_REVISION, + ), + ], +) +def test_real_adapters_require_an_explicit_pinned_cache_path( + adapter_type: type[Qwen3EmbeddingRetriever] | type[Qwen3CandidateReranker], + cache_name: str, + revision: str, + tmp_path: Path, +) -> None: + with pytest.raises(ValueError, match="pinned model cache path is required"): + adapter_type() + + with pytest.raises(ValueError, match="absolute local path"): + adapter_type("Qwen/repository-id") + + with pytest.raises(ValueError, match="pinned revision directory"): + adapter_type(tmp_path / "wrong-model" / "unversioned") + + adapter_type(tmp_path / cache_name / revision) + + +def test_fake_backends_do_not_require_a_model_path() -> None: + Qwen3EmbeddingRetriever(backend=RecordingEmbeddingBackend()) + Qwen3CandidateReranker(backend=FakeLogitBackend(())) + + +def test_sentence_transformer_backend_is_lazy_and_local_only( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + constructor_calls: list[tuple[str, dict[str, Any]]] = [] + + class FakeSentenceTransformer: + max_seq_length = 0 + + def __init__(self, model_path: str, **kwargs: Any) -> None: + constructor_calls.append((model_path, kwargs)) + + def encode(self, texts: Sequence[str], **kwargs: Any) -> list[list[float]]: + assert list(texts) == ["query"] + assert kwargs["batch_size"] == 3 + assert kwargs["normalize_embeddings"] is True + return [[1.0, 0.0]] + + monkeypatch.setitem( + sys.modules, + "sentence_transformers", + SimpleNamespace(SentenceTransformer=FakeSentenceTransformer), + ) + model_path = tmp_path / "embedding" + backend = LocalSentenceTransformerBackend(model_path) + assert constructor_calls == [] + + vectors = backend.encode_queries(("query",), max_length=512, batch_size=3) + + assert vectors == ((1.0, 0.0),) + assert constructor_calls == [(str(model_path), {"local_files_only": True})] + + +def test_reranker_backend_loads_local_only_and_normalizes_yes_no_logits( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + load_calls: list[tuple[str, str, dict[str, Any]]] = [] + model_calls = 0 + inference_mode_active = False + + class FakeInferenceMode: + def __enter__(self) -> None: + nonlocal inference_mode_active + inference_mode_active = True + + def __exit__(self, *args: object) -> None: + nonlocal inference_mode_active + inference_mode_active = False + + class FakeLastTokenLogits: + def __getitem__(self, key: Any) -> FakeLastTokenLogits: + assert key == (slice(None), -1, slice(None)) + return self + + def detach(self) -> FakeLastTokenLogits: + return self + + def cpu(self) -> FakeLastTokenLogits: + return self + + def tolist(self) -> list[list[float]]: + row = [0.0] * 8 + row[5] = 2.0 + row[7] = 0.0 + return [row] + + class FakeTokenizer: + padding_side = "right" + pad_token = None + eos_token = "" + + def convert_tokens_to_ids(self, token: str) -> int: + return {"yes": 5, "no": 7}[token] + + def encode(self, text: str, **kwargs: Any) -> list[int]: + assert kwargs == {"add_special_tokens": False} + if text.startswith("<|im_start|>system"): + return [1, 2] + assert text.startswith("<|im_end|>") + return [6] + + def __call__(self, texts: Sequence[str], **kwargs: Any) -> dict[str, str]: + assert len(texts) == 1 + assert kwargs["max_length"] == 509 + assert kwargs["truncation"] is True + assert kwargs["padding"] is False + return {"input_ids": [[3, 4]]} # type: ignore[dict-item] + + def pad(self, inputs: dict[str, Any], **kwargs: Any) -> dict[str, str]: + assert inputs == {"input_ids": [[1, 2, 3, 4, 6]]} + assert kwargs["padding"] is True + assert kwargs["max_length"] == 512 + return {"input_ids": "fake-inputs"} + + tokenizer = FakeTokenizer() + + class FakeAutoTokenizer: + @staticmethod + def from_pretrained(model_path: str, **kwargs: Any) -> FakeTokenizer: + load_calls.append(("tokenizer", model_path, kwargs)) + return tokenizer + + class FakeModel: + def eval(self) -> FakeModel: + return self + + def __call__(self, **inputs: Any) -> SimpleNamespace: + nonlocal model_calls + assert inference_mode_active is True + assert inputs == {"input_ids": "fake-inputs"} + model_calls += 1 + return SimpleNamespace(logits=FakeLastTokenLogits()) + + class FakeAutoModel: + @staticmethod + def from_pretrained(model_path: str, **kwargs: Any) -> FakeModel: + load_calls.append(("model", model_path, kwargs)) + return FakeModel() + + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace( + AutoModelForCausalLM=FakeAutoModel, + AutoTokenizer=FakeAutoTokenizer, + ), + ) + monkeypatch.setitem( + sys.modules, + "torch", + SimpleNamespace(inference_mode=FakeInferenceMode), + ) + model_path = tmp_path / "reranker" + backend = LocalQwen3RerankerBackend(model_path) + assert load_calls == [] + + scores = backend.score_pairs( + ( + ("회사 연락처", "company.phone"), + ("근로자 연락처", "worker.phone"), + ), + max_length=512, + batch_size=1, + ) + + assert scores == pytest.approx((0.8807970779, 0.8807970779)) + assert model_calls == 2 + assert load_calls == [ + ("tokenizer", str(model_path), {"local_files_only": True}), + ("model", str(model_path), {"local_files_only": True}), + ] + assert tokenizer.padding_side == "left" + assert tokenizer.pad_token == tokenizer.eos_token diff --git a/tests/documents/dynamic_automation/test_rules.py b/tests/documents/dynamic_automation/test_rules.py new file mode 100644 index 0000000..14099ae --- /dev/null +++ b/tests/documents/dynamic_automation/test_rules.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.field_context import build_field_contexts +from app.documents.dynamic_automation.models import DocumentFieldContext +from app.documents.dynamic_automation.rules import classify_non_data, exact_alias_matches + +FIXTURE_DIR = Path(__file__).parents[2] / "fixtures" / "dynamic_automation" +CATALOG_PATH = ( + Path(__file__).parents[3] + / "app" + / "documents" + / "dynamic_automation" + / "resources" + / "canonical_fields.v1.yaml" +) + + +def make_context( + *, label: str, section: str = "", kind: str = "text_field", field_type: str = "text" +) -> DocumentFieldContext: + return DocumentFieldContext( + field_id="field-1", + label=label, + normalized_label=label, + field_type=field_type, + document_title="신청서", + section=section, + row_labels=(), + nearby_labels=(), + options=(), + repeat_index=0, + required=False, + kind=kind, + ) + + +@pytest.mark.parametrize( + ("label", "reason"), + [ + ("확인검토", "process_flow_label"), + ("→", "page_navigation_label"), + ("For Official Use", "official_use_label"), + ], +) +def test_non_data_labels_are_rejected(label: str, reason: str) -> None: + decision = classify_non_data(make_context(label=label, section="처리절차")) + + assert decision.is_non_data is True + assert decision.reason == reason + + +@pytest.mark.parametrize("kind", ["official_region", "signable_region"]) +def test_non_data_regions_are_rejected(kind: str) -> None: + decision = classify_non_data(make_context(label="입력란", kind=kind)) + + assert decision.is_non_data is True + assert decision.reason == kind + + +def test_exact_aliases_return_compatible_candidates_without_substring_matches() -> None: + catalog = CanonicalCatalog.load(CATALOG_PATH) + exact = make_context(label="전화번호", field_type="phone") + substring = make_context(label="전화번호 안내", field_type="phone") + + assert [item.field_id for item in exact_alias_matches(exact, catalog)] == ["company.phone"] + assert exact_alias_matches(substring, catalog) == () + + +def test_extension_registry_rejects_process_flow_labels() -> None: + registry = json.loads( + (FIXTURE_DIR / "extension_application_registry.json").read_text(encoding="utf-8") + ) + contexts = build_field_contexts(registry, document_title="신청서") + + intake = next(item for item in contexts if item.field_id == "intake") + assert classify_non_data(intake).reason == "process_flow_label" diff --git a/tests/documents/dynamic_automation/test_training_dataset.py b/tests/documents/dynamic_automation/test_training_dataset.py new file mode 100644 index 0000000..6fa3f52 --- /dev/null +++ b/tests/documents/dynamic_automation/test_training_dataset.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.feedback import MappingFeedbackRecord +from app.documents.dynamic_automation.qwen import ( + QWEN3_EMBEDDING_CACHE_NAME, + QWEN3_EMBEDDING_REVISION, +) +from app.documents.dynamic_automation.training import ( + build_hard_negatives, + build_training_split, +) +from scripts import train_dynamic_mapping_models as training_cli + +ROOT = Path(__file__).parents[3] +FEEDBACK_PATH = ROOT / "tests/fixtures/dynamic_automation/approved_feedback.jsonl" +CATALOG_PATH = ( + ROOT / "app/documents/dynamic_automation/resources/canonical_fields.v1.yaml" +) + + +def load_feedback_fixture() -> tuple[MappingFeedbackRecord, ...]: + return tuple( + MappingFeedbackRecord.model_validate_json(line) + for line in FEEDBACK_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ) + + +def test_layout_hash_never_crosses_train_and_test() -> None: + split = build_training_split(load_feedback_fixture()) + + train_layouts = {item.document_layout_hash for item in split.train} + test_layouts = {item.document_layout_hash for item in split.test} + + assert train_layouts + assert test_layouts + assert train_layouts.isdisjoint(test_layouts) + + +def test_all_required_group_identities_are_disjoint_across_split() -> None: + split = build_training_split(load_feedback_fixture()) + + for attribute in ( + "document_layout_hash", + "document_kind", + "document_version", + "source_institution", + ): + train_values = {getattr(item, attribute) for item in split.train} + test_values = {getattr(item, attribute) for item in split.test} + assert train_values.isdisjoint(test_values) + + +def test_training_split_keeps_transitively_connected_groups_together() -> None: + base = load_feedback_fixture()[0] + records = ( + base.model_copy( + update={ + "layout_hash": "1" * 64, + "field_context_hash": "1" * 64, + "field_id": "bridge-a", + "document_kind": "shared-kind", + "document_version": "version-a", + "source_institution": "institution-a", + } + ), + base.model_copy( + update={ + "layout_hash": "2" * 64, + "field_context_hash": "2" * 64, + "field_id": "bridge-b", + "document_kind": "shared-kind", + "document_version": "version-b", + "source_institution": "shared-institution", + } + ), + base.model_copy( + update={ + "layout_hash": "3" * 64, + "field_context_hash": "3" * 64, + "field_id": "bridge-c", + "document_kind": "kind-c", + "document_version": "version-c", + "source_institution": "shared-institution", + } + ), + base.model_copy( + update={ + "layout_hash": "4" * 64, + "field_context_hash": "4" * 64, + "field_id": "independent", + "document_kind": "kind-d", + "document_version": "version-d", + "source_institution": "institution-d", + } + ), + ) + + split = build_training_split(records) + train_ids = {item.field_id for item in split.train} + test_ids = {item.field_id for item in split.test} + bridge_ids = {"bridge-a", "bridge-b", "bridge-c"} + + assert bridge_ids <= train_ids or bridge_ids <= test_ids + assert "independent" in train_ids | test_ids + + +def test_training_split_is_reproducible_and_preserves_corrected_labels() -> None: + records = load_feedback_fixture() + + first = build_training_split(records) + second = build_training_split(tuple(reversed(records))) + + assert first == second + corrected = next( + item + for item in (*first.train, *first.test) + if item.field_id == "company_phone" + ) + assert corrected.canonical_field_id == "company.phone" + + +def test_training_split_revalidates_input_and_rejects_db_values() -> None: + payload = load_feedback_fixture()[0].model_dump(mode="json") + payload["db_value"] = "+82-10-1234-5678" + + with pytest.raises(ValueError, match="forbidden key|Extra inputs"): + build_training_split([payload]) + + +def test_feedback_fixture_contains_only_sanitized_mapping_records() -> None: + records = load_feedback_fixture() + raw = [json.loads(line) for line in FEEDBACK_PATH.read_text(encoding="utf-8").splitlines()] + + assert len(records) == 10 + assert all("db_value" not in record and "resolved_value" not in record for record in raw) + + +def test_hard_negatives_are_type_compatible_and_prioritize_entity_confusions() -> None: + worker_phone = next( + record for record in load_feedback_fixture() if record.field_id == "worker_phone" + ) + split = build_training_split([worker_phone]) + catalog = CanonicalCatalog.load(CATALOG_PATH) + + pairs = build_hard_negatives(split, catalog) + + assert pairs + assert pairs[0].positive_canonical_field_id == "worker.phone" + assert pairs[0].negative_canonical_field_id == "company.phone" + assert all( + set(catalog.get(pair.positive_canonical_field_id).compatible_field_types) + & set(catalog.get(pair.negative_canonical_field_id).compatible_field_types) + for pair in pairs + ) + + +def test_passport_and_registration_ids_are_prioritized_hard_negatives() -> None: + passport = next( + record + for record in load_feedback_fixture() + if record.field_id == "passport_identifier" + ) + split = build_training_split([passport]) + catalog = CanonicalCatalog.load(CATALOG_PATH) + + pairs = build_hard_negatives(split, catalog) + + assert pairs[0].positive_canonical_field_id == "identity.passport_number" + assert ( + pairs[0].negative_canonical_field_id + == "identity.alien_registration_number" + ) + + +def test_training_cli_refuses_to_download_when_pinned_cache_is_absent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(tmp_path / "missing-cache")) + output_dir = tmp_path / "output" + + exit_code = training_cli.main( + [ + "--feedback", + str(FEEDBACK_PATH), + "--catalog", + str(CATALOG_PATH), + "--output-dir", + str(output_dir), + "--seed", + "42", + "--model-kind", + "bi-encoder", + ] + ) + + assert exit_code == 1 + assert not output_dir.exists() + + +def test_training_cli_rejects_feedback_from_a_different_catalog_version( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + payloads = [ + record.model_dump(mode="json") for record in load_feedback_fixture() + ] + payloads[-1]["catalog_version"] = "v2" + feedback_path = tmp_path / "mixed-feedback.jsonl" + feedback_path.write_text( + "".join( + json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n" + for payload in payloads + ), + encoding="utf-8", + ) + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(tmp_path / "missing-cache")) + output_dir = tmp_path / "output" + + exit_code = training_cli.main( + [ + "--feedback", + str(feedback_path), + "--catalog", + str(CATALOG_PATH), + "--output-dir", + str(output_dir), + "--seed", + "42", + "--model-kind", + "bi-encoder", + ] + ) + + assert exit_code == 1 + assert "feedback catalog_version v2 does not match loaded catalog v1" in capsys.readouterr().err + assert not output_dir.exists() + + +def test_training_cli_rejects_config_only_cache_without_model_weights( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + model_cache = tmp_path / "model-cache" + pinned_model = model_cache / QWEN3_EMBEDDING_CACHE_NAME / QWEN3_EMBEDDING_REVISION + pinned_model.mkdir(parents=True) + (pinned_model / "config.json").write_text("{}\n", encoding="utf-8") + monkeypatch.setenv("FOWOCO_MODEL_CACHE_DIR", str(model_cache)) + output_dir = tmp_path / "output" + + exit_code = training_cli.main( + [ + "--feedback", + str(FEEDBACK_PATH), + "--catalog", + str(CATALOG_PATH), + "--output-dir", + str(output_dir), + "--seed", + "42", + "--model-kind", + "bi-encoder", + ] + ) + + assert exit_code == 1 + assert not output_dir.exists() + + +def test_training_script_refuses_config_only_cache_without_import_path_errors( + tmp_path: Path, +) -> None: + model_cache = tmp_path / "model-cache" + pinned_model = model_cache / QWEN3_EMBEDDING_CACHE_NAME / QWEN3_EMBEDDING_REVISION + pinned_model.mkdir(parents=True) + (pinned_model / "config.json").write_text("{}\n", encoding="utf-8") + output_dir = tmp_path / "subprocess-output" + environment = os.environ.copy() + environment["FOWOCO_MODEL_CACHE_DIR"] = str(model_cache) + + completed = subprocess.run( + [ + sys.executable, + "scripts/train_dynamic_mapping_models.py", + "--feedback", + str(FEEDBACK_PATH), + "--catalog", + str(CATALOG_PATH), + "--output-dir", + str(output_dir), + "--seed", + "42", + "--model-kind", + "bi-encoder", + ], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 1 + assert "ModuleNotFoundError" not in completed.stderr + assert "model weights" in completed.stderr + assert not output_dir.exists() diff --git a/tests/fixtures/dynamic_automation/approved_feedback.jsonl b/tests/fixtures/dynamic_automation/approved_feedback.jsonl new file mode 100644 index 0000000..cc04ca4 --- /dev/null +++ b/tests/fixtures/dynamic_automation/approved_feedback.jsonl @@ -0,0 +1,10 @@ +{"candidate_scores":[{"canonical_field_id":"worker.phone","rank":1,"score":0.99},{"canonical_field_id":"company.phone","rank":2,"score":0.85}],"catalog_version":"v1","decision":"accepted","document_kind":"worker_contact_form","document_version":"v1","field_context_hash":"0000000000000000000000000000000000000000000000000000000000000000","field_id":"worker_phone","final_canonical_field_id":"worker.phone","label":"Worker phone","layout_hash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","model_version":"qwen-test-v1","nearby_labels":["Worker details"],"predicted_canonical_field_id":"worker.phone","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Worker"],"schema_version":"v2","section":"Worker","source_institution":"labor_office"} +{"candidate_scores":[{"canonical_field_id":"company.phone","rank":1,"score":0.98},{"canonical_field_id":"worker.phone","rank":2,"score":0.82}],"catalog_version":"v1","decision":"corrected","document_kind":"worker_contact_form","document_version":"v1","field_context_hash":"1111111111111111111111111111111111111111111111111111111111111111","field_id":"company_phone","final_canonical_field_id":"company.phone","label":"Company phone","layout_hash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","model_version":"qwen-test-v1","nearby_labels":["Company details"],"predicted_canonical_field_id":"worker.phone","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Company"],"schema_version":"v2","section":"Company","source_institution":"labor_office"} +{"candidate_scores":[{"canonical_field_id":"worker.legal_name","rank":1,"score":0.97},{"canonical_field_id":"company.representative_name","rank":2,"score":0.8}],"catalog_version":"v1","decision":"accepted","document_kind":"employment_roster","document_version":"v2","field_context_hash":"2222222222222222222222222222222222222222222222222222222222222222","field_id":"worker_name","final_canonical_field_id":"worker.legal_name","label":"Worker name","layout_hash":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","model_version":"qwen-test-v1","nearby_labels":["Applicant"],"predicted_canonical_field_id":"worker.legal_name","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Worker"],"schema_version":"v2","section":"Applicant","source_institution":"local_center"} +{"candidate_scores":[{"canonical_field_id":"company.representative_name","rank":1,"score":0.96},{"canonical_field_id":"worker.legal_name","rank":2,"score":0.78}],"catalog_version":"v1","decision":"accepted","document_kind":"employment_roster","document_version":"v2","field_context_hash":"3333333333333333333333333333333333333333333333333333333333333333","field_id":"representative_name","final_canonical_field_id":"company.representative_name","label":"Representative name","layout_hash":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","model_version":"qwen-test-v1","nearby_labels":["Employer"],"predicted_canonical_field_id":"company.representative_name","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Company"],"schema_version":"v2","section":"Employer","source_institution":"local_center"} +{"candidate_scores":[{"canonical_field_id":"identity.passport_number","rank":1,"score":0.99},{"canonical_field_id":"identity.alien_registration_number","rank":2,"score":0.71}],"catalog_version":"v1","decision":"accepted","document_kind":"identity_form","document_version":"v3","field_context_hash":"4444444444444444444444444444444444444444444444444444444444444444","field_id":"passport_identifier","final_canonical_field_id":"identity.passport_number","label":"Passport number","layout_hash":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","model_version":"qwen-test-v1","nearby_labels":["Identity"],"predicted_canonical_field_id":"identity.passport_number","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Worker"],"schema_version":"v2","section":"Identity","source_institution":"immigration_office"} +{"candidate_scores":[{"canonical_field_id":"identity.alien_registration_number","rank":1,"score":0.98},{"canonical_field_id":"identity.passport_number","rank":2,"score":0.73}],"catalog_version":"v1","decision":"accepted","document_kind":"identity_form","document_version":"v3","field_context_hash":"5555555555555555555555555555555555555555555555555555555555555555","field_id":"resident_identifier","final_canonical_field_id":"identity.alien_registration_number","label":"Alien registration number","layout_hash":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","model_version":"qwen-test-v1","nearby_labels":["Identity"],"predicted_canonical_field_id":"identity.alien_registration_number","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Worker"],"schema_version":"v2","section":"Identity","source_institution":"immigration_office"} +{"candidate_scores":[{"canonical_field_id":"contract.start_date","rank":1,"score":0.95},{"canonical_field_id":"application.date","rank":2,"score":0.79}],"catalog_version":"v1","decision":"accepted","document_kind":"contract_form","document_version":"v4","field_context_hash":"6666666666666666666666666666666666666666666666666666666666666666","field_id":"contract_start","final_canonical_field_id":"contract.start_date","label":"Contract start date","layout_hash":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","model_version":"qwen-test-v1","nearby_labels":["Employment term"],"predicted_canonical_field_id":"contract.start_date","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Contract"],"schema_version":"v2","section":"Contract","source_institution":"labor_board"} +{"candidate_scores":[{"canonical_field_id":"application.date","rank":1,"score":0.94},{"canonical_field_id":"contract.start_date","rank":2,"score":0.77}],"catalog_version":"v1","decision":"accepted","document_kind":"contract_form","document_version":"v4","field_context_hash":"7777777777777777777777777777777777777777777777777777777777777777","field_id":"application_date","final_canonical_field_id":"application.date","label":"Application date","layout_hash":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","model_version":"qwen-test-v1","nearby_labels":["Submission"],"predicted_canonical_field_id":"application.date","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Application"],"schema_version":"v2","section":"Application","source_institution":"labor_board"} +{"candidate_scores":[{"canonical_field_id":"company.name","rank":1,"score":0.97}],"catalog_version":"v1","decision":"accepted","document_kind":"company_form","document_version":"v5","field_context_hash":"8888888888888888888888888888888888888888888888888888888888888888","field_id":"company_name","final_canonical_field_id":"company.name","label":"Company name","layout_hash":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","model_version":"qwen-test-v1","nearby_labels":["Employer"],"predicted_canonical_field_id":"company.name","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Company"],"schema_version":"v2","section":"Employer","source_institution":"business_office"} +{"candidate_scores":[{"canonical_field_id":"company.address","rank":1,"score":0.96}],"catalog_version":"v1","decision":"accepted","document_kind":"company_form","document_version":"v5","field_context_hash":"9999999999999999999999999999999999999999999999999999999999999999","field_id":"company_address","final_canonical_field_id":"company.address","label":"Company address","layout_hash":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","model_version":"qwen-test-v1","nearby_labels":["Employer"],"predicted_canonical_field_id":"company.address","predicted_status":"MATCHED","repeat_index":0,"row_labels":["Company"],"schema_version":"v2","section":"Employer","source_institution":"business_office"} diff --git a/tests/fixtures/dynamic_automation/extension_application_registry.json b/tests/fixtures/dynamic_automation/extension_application_registry.json new file mode 100644 index 0000000..1f42539 --- /dev/null +++ b/tests/fixtures/dynamic_automation/extension_application_registry.json @@ -0,0 +1,70 @@ +[ + { + "field_id": "intake", + "target_id": "section0.table0.row1.cell0", + "label": "접수", + "type": "text", + "category": "step1_application", + "row": 1, + "column": 0, + "current_text": "", + "required": false, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row1.cell0"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "next-page", + "target_id": "section0.table0.row1.cell1", + "label": "→", + "type": "text", + "category": "step1_application", + "row": 1, + "column": 1, + "current_text": "", + "required": false, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row1.cell1"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "official-use", + "target_id": "section0.table0.row2.cell0", + "label": "For Official Use", + "type": "text", + "category": "step1_application", + "row": 2, + "column": 0, + "current_text": "", + "required": false, + "options": null, + "kind": "official_region", + "xml_segments": ["section0.table0.row2.cell0"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "signature", + "target_id": "section0.table0.row3.cell0", + "label": "서명", + "type": "signature", + "category": "step1_application", + "row": 3, + "column": 0, + "current_text": "", + "required": true, + "options": null, + "kind": "signable_region", + "xml_segments": ["section0.table0.row3.cell0"], + "visual_regions": [], + "constraints": {}, + "disposition": null + } +] diff --git a/tests/fixtures/dynamic_automation/integrated_application_registry.json b/tests/fixtures/dynamic_automation/integrated_application_registry.json new file mode 100644 index 0000000..bb9c6c1 --- /dev/null +++ b/tests/fixtures/dynamic_automation/integrated_application_registry.json @@ -0,0 +1,104 @@ +[ + { + "field_id": "workplace-heading", + "target_id": "section0.table0.row7.cell0", + "label": "현재 근무처", + "type": "text", + "category": "step1_application", + "row": 7, + "column": 0, + "current_text": "", + "required": false, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row7.cell0"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "business-registration-number", + "target_id": "section0.table0.row7.cell1", + "label": "사업자등록번호", + "type": "number", + "category": "step1_application", + "row": 7, + "column": 1, + "current_text": "", + "required": true, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row7.cell1"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "workplace-phone", + "target_id": "section0.table0.row7.cell2", + "label": "전화번호:", + "type": "phone", + "category": "step1_application", + "row": 7, + "column": 2, + "current_text": "", + "required": true, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row7.cell2"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "worker-heading", + "target_id": "section0.table0.row12.cell0", + "label": "근로자", + "type": "text", + "category": "step1_application", + "row": 12, + "column": 0, + "current_text": "", + "required": false, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row12.cell0"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "worker-phone", + "target_id": "section0.table0.row12.cell1", + "label": "전화번호", + "type": "phone", + "category": "step1_application", + "row": 12, + "column": 1, + "current_text": "", + "required": true, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row12.cell1"], + "visual_regions": [], + "constraints": {}, + "disposition": null + }, + { + "field_id": "alien-registration-number", + "target_id": "section0.table0.row13.cell0", + "label": "외국인등록번호", + "type": "number", + "category": "step2_personal", + "row": 13, + "column": 0, + "current_text": "", + "required": true, + "options": null, + "kind": "text_field", + "xml_segments": ["section0.table0.row13.cell0"], + "visual_regions": [], + "constraints": {}, + "disposition": null + } +] diff --git a/tests/fixtures/dynamic_automation/mapping_cases.jsonl b/tests/fixtures/dynamic_automation/mapping_cases.jsonl new file mode 100644 index 0000000..a216bda --- /dev/null +++ b/tests/fixtures/dynamic_automation/mapping_cases.jsonl @@ -0,0 +1,6 @@ +{"case_id":"company-name","document_id":"application-a","context":{"field_id":"company-name-1","label":"Company name","normalized_label":"companyname","field_type":"text","document_title":"Employment application","section":"Company","row_labels":["Company","Company name"],"nearby_labels":[],"options":[],"repeat_index":0,"required":true,"kind":"text_field"},"expected_status":"MATCHED","expected_canonical_field_id":"company.name"} +{"case_id":"passport-number","document_id":"application-a","context":{"field_id":"passport-1","label":"Passport number","normalized_label":"passportnumber","field_type":"text","document_title":"Employment application","section":"Identity","row_labels":["Identity","Passport number"],"nearby_labels":[],"options":[],"repeat_index":0,"required":true,"kind":"text_field"},"expected_status":"MATCHED","expected_canonical_field_id":"identity.passport_number"} +{"case_id":"generic-phone","document_id":"application-b","context":{"field_id":"phone-1","label":"Phone number","normalized_label":"phonenumber","field_type":"phone","document_title":"Employment application","section":"Contact","row_labels":["Contact","Phone number"],"nearby_labels":[],"options":[],"repeat_index":0,"required":true,"kind":"text_field"},"expected_status":"AMBIGUOUS","expected_canonical_field_id":null} +{"case_id":"semantic-company-contact","document_id":"application-b","context":{"field_id":"company-contact-1","label":"Employer contact line","normalized_label":"employercontactline","field_type":"phone","document_title":"Employment application","section":"Employer details","row_labels":["Employer details","Contact line"],"nearby_labels":["Business registration"],"options":[],"repeat_index":0,"required":true,"kind":"text_field"},"expected_status":"MATCHED","expected_canonical_field_id":"company.phone"} +{"case_id":"official-region","document_id":"application-b","context":{"field_id":"official-1","label":"For Official Use","normalized_label":"forofficialuse","field_type":"text","document_title":"Employment application","section":"Administration","row_labels":["For Official Use"],"nearby_labels":[],"options":[],"repeat_index":0,"required":false,"kind":"official_region"},"expected_status":"NON_DATA","expected_canonical_field_id":null} +{"case_id":"unsupported-field","document_id":"application-c","context":{"field_id":"unsupported-1","label":"Internal barcode","normalized_label":"internalbarcode","field_type":"placeholder","document_title":"Employment application","section":"Administration","row_labels":["Internal barcode"],"nearby_labels":[],"options":[],"repeat_index":0,"required":false,"kind":"placeholder"},"expected_status":"UNMAPPED","expected_canonical_field_id":null} diff --git a/tests/integration/dynamic_automation/test_qwen_mapping_smoke.py b/tests/integration/dynamic_automation/test_qwen_mapping_smoke.py new file mode 100644 index 0000000..7e70e60 --- /dev/null +++ b/tests/integration/dynamic_automation/test_qwen_mapping_smoke.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import socket +from pathlib import Path + +import pytest + +from app.documents.dynamic_automation.catalog import CanonicalCatalog +from app.documents.dynamic_automation.config import DynamicAutomationSettings +from app.documents.dynamic_automation.models import DocumentFieldContext +from app.documents.dynamic_automation.qwen import ( + Qwen3CandidateReranker, + Qwen3EmbeddingRetriever, +) + + +@pytest.mark.language_models +def test_qwen_mapping_models_run_from_pinned_local_cache_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = DynamicAutomationSettings(_env_file=None) + embedding_path = settings.dynamic_automation_embedding_model_path + reranker_path = settings.dynamic_automation_reranker_model_path + assert embedding_path is not None + assert reranker_path is not None + if not embedding_path.is_dir() or not reranker_path.is_dir(): + pytest.skip("pinned Qwen3 mapping model cache is not provisioned") + + def reject_network(*args: object, **kwargs: object) -> None: + del args, kwargs + raise AssertionError("Qwen3 mapping smoke attempted outbound network access") + + monkeypatch.setattr(socket.socket, "connect", reject_network) + catalog_path = ( + Path(__file__).parents[3] + / "app" + / "documents" + / "dynamic_automation" + / "resources" + / "canonical_fields.v1.yaml" + ) + catalog = CanonicalCatalog.load(catalog_path) + context = DocumentFieldContext( + field_id="field-1", + label="회사 연락처", + normalized_label="회사 연락처", + field_type="phone", + document_title="통합신청서", + section="현재 근무처", + row_labels=("현재 근무처", "연락처"), + nearby_labels=("회사명",), + options=(), + repeat_index=0, + required=True, + kind="text_field", + ) + candidates = catalog.compatible(context) + + retrieved = Qwen3EmbeddingRetriever(embedding_path).retrieve( + context, + candidates, + top_k=2, + ) + definitions_by_id = {definition.field_id: definition for definition in candidates} + reranked = Qwen3CandidateReranker( + reranker_path, + definition_resolver=definitions_by_id.__getitem__, + ).rerank(context, retrieved) + + assert len(retrieved) == 2 + assert len(reranked) == 2 + assert all(0 <= candidate.score <= 1 for candidate in reranked) diff --git a/uv.lock b/uv.lock index 6e3fb80..62fe597 100644 --- a/uv.lock +++ b/uv.lock @@ -528,6 +528,12 @@ dev = [ { name = "pytest-asyncio" }, { name = "ruff" }, ] +document-automation = [ + { name = "pyyaml" }, + { name = "sentence-transformers" }, + { name = "torch" }, + { name = "transformers" }, +] intent = [ { name = "accelerate" }, { name = "torch" }, @@ -567,13 +573,17 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25,<1" }, { name = "python-dotenv", specifier = ">=1.0,<2" }, { name = "python-multipart", specifier = ">=0.0.20,<1" }, + { name = "pyyaml", marker = "extra == 'document-automation'", specifier = ">=6.0,<7" }, { name = "qdrant-client", marker = "extra == 'language-retrieval'", specifier = ">=1.19,<2" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12,<1" }, + { name = "sentence-transformers", marker = "extra == 'document-automation'", specifier = ">=5,<6" }, + { name = "torch", marker = "extra == 'document-automation'", specifier = ">=2.2,<3" }, { name = "torch", marker = "extra == 'intent'", specifier = ">=2.2,<3" }, + { name = "transformers", marker = "extra == 'document-automation'", specifier = ">=4.51,<5" }, { name = "transformers", marker = "extra == 'intent'", specifier = ">=4.46,<5" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34,<1" }, ] -provides-extras = ["dev", "knowledge", "intent", "intent-ax", "language-retrieval"] +provides-extras = ["dev", "knowledge", "intent", "intent-ax", "language-retrieval", "document-automation"] [[package]] name = "frozenlist"