diff --git a/Makefile b/Makefile index ac6cc411..2426a66f 100644 --- a/Makefile +++ b/Makefile @@ -357,6 +357,7 @@ help-full: @echo " export SEC_USER_AGENT='Name email@example.com'" @echo " make sec-stage TICKERS=NVDA,MSFT" @echo " make sec-fundamentals-preview TICKERS=AAPL,NVDA,AMD Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" + @echo " make sec-fundamentals-patch-preview SEC_PREVIEW=/tmp/reviewed-sec-preview.json EXPECTED_SEC_PREVIEW_SHA256= EXPECTED_CANONICAL_SHA256= Exact reviewed-cell projection; drift fails closed; no writes or apply" @echo " make yfinance-stage TICKERS=NVDA" @echo " make fundamentals-source-ladder TICKERS=NVDA" @echo " Try SEC, yfinance, FMP, Alpha Vantage, then Finnhub before stopping at reviewed blocker evidence" @@ -1286,6 +1287,19 @@ ifndef TICKERS endif @PYTHONDONTWRITEBYTECODE=1 python3 -m src.sec_fundamentals_preview --tickers "$(TICKERS)" +.PHONY: sec-fundamentals-patch-preview +sec-fundamentals-patch-preview: +ifndef SEC_PREVIEW + $(error SEC_PREVIEW is required, for example: make sec-fundamentals-patch-preview SEC_PREVIEW=/tmp/reviewed-sec-preview.json) +endif +ifndef EXPECTED_SEC_PREVIEW_SHA256 + $(error EXPECTED_SEC_PREVIEW_SHA256 is required) +endif +ifndef EXPECTED_CANONICAL_SHA256 + $(error EXPECTED_CANONICAL_SHA256 is required) +endif + @PYTHONDONTWRITEBYTECODE=1 python3 -m src.sec_fundamentals_patch_preview --sec-preview-path "$(SEC_PREVIEW)" --canonical-path "$(or $(CANONICAL_PATH),data/fundamentals.csv)" --expected-sec-preview-sha256 "$(EXPECTED_SEC_PREVIEW_SHA256)" --expected-canonical-sha256 "$(EXPECTED_CANONICAL_SHA256)" --repository-head "$(shell git rev-parse HEAD)" + demo-dashboard-render-smoke: @STOCK_RESEARCH_DATA_PROFILE=demo python3 -m src.dashboard_render_smoke diff --git a/src/sec_fundamentals_patch_preview.py b/src/sec_fundamentals_patch_preview.py new file mode 100644 index 00000000..0e6b37ad --- /dev/null +++ b/src/sec_fundamentals_patch_preview.py @@ -0,0 +1,465 @@ +"""Pure, no-write preview for a reviewed SEC fundamentals cell patch.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import json +import re +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping + + +_EXPECTED_TICKERS = ("AAPL", "NVDA", "AMD") +_EXPECTED_CIKS = { + "AAPL": "0000320193", + "NVDA": "0001045810", + "AMD": "0000002488", +} +_ALLOWED_FIELDS = { + "revenue": "revenue", + "filing_dates": "sec_filed_date", +} +_SEC_URL_PREFIX = "https://data.sec.gov/api/xbrl/companyfacts/CIK" +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_GIT_HEAD_PATTERN = re.compile(r"^[0-9a-f]{40}$") + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json_bytes(value: Any) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _parse_packet(value: bytes) -> Mapping[str, Any]: + try: + packet = json.loads(value) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("SEC preview packet must be valid JSON") from exc + if not isinstance(packet, Mapping): + raise ValueError("SEC preview packet must be a JSON object") + return packet + + +def _parse_canonical(value: bytes) -> tuple[list[str], list[list[str]]]: + try: + decoded = value.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("canonical CSV must be UTF-8") from exc + rows = list(csv.reader(io.StringIO(decoded, newline=""))) + if not rows or not rows[0]: + raise ValueError("canonical CSV header is required") + header = rows[0] + if len(header) != len(set(header)): + raise ValueError("canonical CSV columns must be unique") + for row in rows[1:]: + if len(row) != len(header): + raise ValueError("canonical row width must match the header") + if "ticker" not in header: + raise ValueError("canonical CSV requires ticker") + for column in _ALLOWED_FIELDS.values(): + if column not in header: + raise ValueError(f"canonical schema is missing reviewed column: {column}") + return header, rows[1:] + + +def _values_equal(canonical: str, expected: Any) -> bool: + if expected is None: + return canonical == "" + if isinstance(expected, bool): + return canonical.lower() == str(expected).lower() + if isinstance(expected, (int, float)): + try: + return Decimal(canonical) == Decimal(str(expected)) + except InvalidOperation: + return False + return canonical == str(expected) + + +def _official_sec_url(value: Any) -> bool: + return ( + isinstance(value, str) + and value.startswith(_SEC_URL_PREFIX) + and value.endswith(".json") + and "?" not in value + and "#" not in value + ) + + +def _validate_field( + field: Mapping[str, Any], + ticker: str, + name: str, + *, + sec_cik: str, + retrieval_timestamp: str, +) -> None: + if field.get("ticker") != ticker or field.get("field") != name: + raise ValueError("candidate field scope does not match its ticker") + if field.get("canonical_column") != _ALLOWED_FIELDS[name]: + raise ValueError("candidate field does not map to the reviewed canonical column") + required = { + "classification": "approved_direct", + "commercial_rights_approved": True, + "field_scope_status": "approved", + "schema_status": "existing_canonical", + "publishability_blocker": "none", + "value_kind": "direct", + "value_status": "changed", + "source_rights_status": "approved", + } + for key, expected in required.items(): + if field.get(key) != expected: + if key == "field_scope_status": + raise ValueError("candidate requires approved field scope") + raise ValueError(f"candidate field failed reviewed evidence contract: {key}") + expected_source_url = f"{_SEC_URL_PREFIX}{sec_cik}.json" + if field.get("sec_cik") != sec_cik: + raise ValueError("candidate field does not match the reviewed SEC CIK") + if field.get("source_url") != expected_source_url: + raise ValueError("candidate must use an official SEC source URL") + if field.get("retrieval_timestamp") != retrieval_timestamp: + raise ValueError("candidate retrieval timestamp does not match the SEC packet") + refs = field.get("source_refs") + if not isinstance(refs, list) or not refs: + raise ValueError("candidate must include SEC provenance") + for ref in refs: + if not isinstance(ref, Mapping) or not _official_sec_url(ref.get("source_url")): + raise ValueError("candidate must use official SEC source references") + if ref.get("source_url") != expected_source_url: + raise ValueError("candidate source reference does not match the reviewed SEC CIK") + if ref.get("retrieval_timestamp") != retrieval_timestamp: + raise ValueError("candidate source reference retrieval timestamp mismatch") + for key in ("accession", "concept", "taxonomy", "form", "filed", "period_end"): + if not ref.get(key): + raise ValueError(f"candidate SEC provenance is missing {key}") + if name == "filing_dates": + if field.get("unit") != "date" or field.get("source_units") != ["date"]: + raise ValueError("filing date provenance must use date units") + if any(ref.get("unit") != "date" for ref in refs): + raise ValueError("filing date source references must use date units") + if any(ref.get("underlying_fact_unit") != "USD" for ref in refs): + raise ValueError("filing date provenance must preserve the underlying fact unit") + elif field.get("unit") != "USD" or field.get("source_units") != ["USD"]: + raise ValueError("revenue provenance must use USD units") + elif any(ref.get("unit") != "USD" for ref in refs): + raise ValueError("revenue source references must use USD units") + + +def _cell_hash_payload( + header: list[str], rows: list[list[str]], excluded: set[tuple[int, int]] +) -> list[list[Any]]: + return [ + [row_index, header[column_index], value] + for row_index, row in enumerate(rows) + for column_index, value in enumerate(row) + if (row_index, column_index) not in excluded + ] + + +def build_sec_fundamentals_patch_preview( + sec_preview_bytes: bytes, + canonical_csv_bytes: bytes, + *, + canonical_path: str, + expected_sec_preview_sha256: str, + expected_canonical_sha256: str, + repository_head: str, +) -> dict[str, Any]: + if not _SHA256_PATTERN.fullmatch(expected_sec_preview_sha256): + raise ValueError("expected SEC preview SHA-256 is invalid") + if not _SHA256_PATTERN.fullmatch(expected_canonical_sha256): + raise ValueError("expected canonical SHA-256 is invalid") + if not _GIT_HEAD_PATTERN.fullmatch(repository_head): + raise ValueError("repository HEAD must be a full lowercase Git hash") + sec_preview_sha256 = _sha256(sec_preview_bytes) + canonical_sha256 = _sha256(canonical_csv_bytes) + if sec_preview_sha256 != expected_sec_preview_sha256: + raise ValueError("SEC preview hash precondition mismatch") + if canonical_sha256 != expected_canonical_sha256: + raise ValueError("canonical hash precondition mismatch") + packet = _parse_packet(sec_preview_bytes) + if packet.get("status") != "inspection_only": + raise ValueError("SEC preview must remain inspection_only") + if packet.get("canonical_apply_authorized") is not False: + raise ValueError("SEC preview must not carry apply authorization") + if packet.get("repository_writes") != []: + raise ValueError("SEC preview must report no repository writes") + if packet.get("source") != "sec_companyfacts": + raise ValueError("SEC preview must use sec_companyfacts only") + if packet.get("source_rights_mutated") is not False: + raise ValueError("SEC preview must not mutate source rights") + if packet.get("requested_tickers") != list(_EXPECTED_TICKERS): + raise ValueError("SEC preview must use the reviewed AAPL,NVDA,AMD cohort") + retrieval_timestamp = packet.get("retrieval_timestamp") + if not isinstance(retrieval_timestamp, str) or not retrieval_timestamp: + raise ValueError("SEC preview retrieval timestamp is required") + + ticker_rows = packet.get("tickers") + if not isinstance(ticker_rows, list): + raise ValueError("SEC preview ticker rows are required") + by_ticker: dict[str, Mapping[str, Any]] = {} + for row in ticker_rows: + if not isinstance(row, Mapping) or not isinstance(row.get("ticker"), str): + raise ValueError("SEC preview ticker row is invalid") + ticker = row["ticker"] + if ticker in by_ticker: + raise ValueError(f"duplicate ticker row: {ticker}") + by_ticker[ticker] = row + if set(by_ticker) != set(_EXPECTED_TICKERS): + raise ValueError("SEC preview ticker rows do not match the reviewed cohort") + + header, rows = _parse_canonical(canonical_csv_bytes) + ticker_column = header.index("ticker") + canonical_by_ticker: dict[str, tuple[int, list[str]]] = {} + for index, row in enumerate(rows): + ticker = row[ticker_column].strip().upper() + if ticker in canonical_by_ticker: + raise ValueError(f"duplicate canonical ticker row: {ticker}") + canonical_by_ticker[ticker] = (index, row) + + patch_cells: list[dict[str, Any]] = [] + coordinates: set[tuple[int, int]] = set() + projected_rows = [list(row) for row in rows] + for ticker in _EXPECTED_TICKERS: + row = by_ticker[ticker] + sec_cik = row.get("sec_cik") + if not isinstance(sec_cik, str) or not sec_cik.isdigit() or len(sec_cik) != 10: + raise ValueError(f"reviewed SEC CIK is invalid for {ticker}") + if sec_cik != _EXPECTED_CIKS[ticker]: + raise ValueError(f"reviewed ticker-to-CIK mapping mismatch for {ticker}") + candidates = row.get("future_apply_candidate_fields") + if not isinstance(candidates, list) or len(candidates) != len(set(candidates)): + raise ValueError(f"invalid future candidate fields for {ticker}") + unexpected = sorted(set(candidates) - set(_ALLOWED_FIELDS)) + if unexpected: + raise ValueError(f"unexpected candidate field: {unexpected[0]}") + if ticker == "NVDA" and candidates: + raise ValueError("NVDA must not have changed future candidates") + if ticker in {"AAPL", "AMD"} and candidates != ["revenue", "filing_dates"]: + raise ValueError(f"{ticker} must have exactly the reviewed candidate fields") + fields = row.get("fields") + if not isinstance(fields, list): + raise ValueError(f"field evidence is required for {ticker}") + field_lookup: dict[str, Mapping[str, Any]] = {} + for field in fields: + if not isinstance(field, Mapping) or not isinstance(field.get("field"), str): + raise ValueError(f"invalid field evidence for {ticker}") + name = field["field"] + if name in field_lookup: + raise ValueError(f"duplicate field evidence: {ticker}:{name}") + field_lookup[name] = field + if ticker not in canonical_by_ticker: + raise ValueError(f"canonical ticker row is missing: {ticker}") + row_index, canonical_row = canonical_by_ticker[ticker] + for name in candidates: + field = field_lookup.get(name) + if field is None: + raise ValueError(f"candidate field evidence is missing: {ticker}:{name}") + _validate_field( + field, + ticker, + name, + sec_cik=sec_cik, + retrieval_timestamp=retrieval_timestamp, + ) + column = _ALLOWED_FIELDS[name] + column_index = header.index(column) + coordinate = (row_index, column_index) + if coordinate in coordinates: + raise ValueError(f"duplicate patch cell: {ticker}:{column}") + current_value = canonical_row[column_index] + if not _values_equal(current_value, field.get("canonical_value")): + raise ValueError(f"canonical precondition mismatch: {ticker}:{column}") + candidate_value = field.get("candidate_value") + if _values_equal(current_value, candidate_value): + raise ValueError(f"changed candidate does not change canonical value: {ticker}:{column}") + refs = field["source_refs"] + primary_ref = refs[0] + provenance_sha256 = _sha256(_json_bytes(refs)) + patch_cells.append( + { + "ticker": ticker, + "field": name, + "canonical_column": column, + "canonical_precondition": field.get("canonical_value"), + "candidate_value": candidate_value, + "unit": field.get("unit"), + "commercial_rights_approved": field.get( + "commercial_rights_approved" + ), + "source_rights_status": field.get("source_rights_status"), + "field_scope_status": field.get("field_scope_status"), + "schema_status": field.get("schema_status"), + "retrieval_timestamp": field.get("retrieval_timestamp"), + "source_url": field.get("source_url"), + "period_start": primary_ref.get("period_start"), + "period_end": primary_ref.get("period_end"), + "filing_date": primary_ref.get("filed"), + "accession": primary_ref.get("accession"), + "concept": primary_ref.get("concept"), + "taxonomy": primary_ref.get("taxonomy"), + "form": primary_ref.get("form"), + "source_refs": refs, + "provenance_sha256": provenance_sha256, + } + ) + projected_rows[row_index][column_index] = str(candidate_value) + coordinates.add(coordinate) + + before_untouched = _sha256(_json_bytes(_cell_hash_payload(header, rows, coordinates))) + after_untouched = _sha256( + _json_bytes(_cell_hash_payload(header, projected_rows, coordinates)) + ) + if before_untouched != after_untouched: + raise ValueError("unrelated canonical cells changed in memory") + touched_row_indexes = {row_index for row_index, _ in coordinates} + touched_column_indexes = {column_index for _, column_index in coordinates} + before_untouched_rows = _sha256( + _json_bytes( + [ + [row_index, row] + for row_index, row in enumerate(rows) + if row_index not in touched_row_indexes + ] + ) + ) + after_untouched_rows = _sha256( + _json_bytes( + [ + [row_index, row] + for row_index, row in enumerate(projected_rows) + if row_index not in touched_row_indexes + ] + ) + ) + before_untouched_columns = _sha256( + _json_bytes( + [ + [header[column_index], [row[column_index] for row in rows]] + for column_index in range(len(header)) + if column_index not in touched_column_indexes + ] + ) + ) + after_untouched_columns = _sha256( + _json_bytes( + [ + [ + header[column_index], + [row[column_index] for row in projected_rows], + ] + for column_index in range(len(header)) + if column_index not in touched_column_indexes + ] + ) + ) + if before_untouched_rows != after_untouched_rows: + raise ValueError("unrelated canonical rows changed in memory") + if before_untouched_columns != after_untouched_columns: + raise ValueError("unrelated canonical columns changed in memory") + identity_inputs = { + "canonical_sha256": canonical_sha256, + "sec_preview_sha256": sec_preview_sha256, + "repository_head": repository_head, + "patch_coordinates": [ + [cell["ticker"], cell["canonical_column"]] for cell in patch_cells + ], + } + return { + "status": "inspection_only", + "canonical_apply_authorized": False, + "repository_writes": [], + "source_rights_mutated": False, + "readiness_mutated": False, + "materialization_performed": False, + "changed_cell_count": len(patch_cells), + "patch_cells": patch_cells, + "preconditions": { + "canonical_path": canonical_path, + "canonical_sha256": canonical_sha256, + "sec_preview_sha256": sec_preview_sha256, + "repository_head": repository_head, + }, + "projection_identity": _sha256(_json_bytes(identity_inputs)), + "in_memory_projection_proof": { + "column_count_before": len(header), + "column_count_after": len(header), + "schema_added_columns": [], + "schema_removed_columns": [], + "row_count_before": len(rows), + "row_count_after": len(rows), + "row_order_unchanged": True, + "full_row_replacement": False, + "staged_input_used": False, + "untouched_cells_unchanged": True, + "untouched_cells_sha256_before": before_untouched, + "untouched_cells_sha256_after": after_untouched, + "untouched_rows_unchanged": True, + "untouched_rows_sha256_before": before_untouched_rows, + "untouched_rows_sha256_after": after_untouched_rows, + "untouched_columns_unchanged": True, + "untouched_columns_sha256_before": before_untouched_columns, + "untouched_columns_sha256_after": after_untouched_columns, + "projected_semantic_matrix_sha256": _sha256( + _json_bytes({"header": header, "rows": projected_rows}) + ), + }, + "next_owner_decision": ( + "Separately authorize or reject only these four hash-bound cells; " + "regenerate this inspection-only preview if repository, canonical, or SEC evidence bytes drift." + ), + } + + +def render_sec_fundamentals_patch_preview(result: Mapping[str, Any]) -> str: + return json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Project a reviewed SEC direct-field patch in memory without writes." + ) + parser.add_argument("--sec-preview-path", type=Path, required=True) + parser.add_argument( + "--canonical-path", + type=Path, + default=Path("data/fundamentals.csv"), + ) + parser.add_argument("--expected-sec-preview-sha256", required=True) + parser.add_argument("--expected-canonical-sha256", required=True) + parser.add_argument("--repository-head", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + try: + result = build_sec_fundamentals_patch_preview( + args.sec_preview_path.read_bytes(), + args.canonical_path.read_bytes(), + canonical_path=str(args.canonical_path), + expected_sec_preview_sha256=args.expected_sec_preview_sha256, + expected_canonical_sha256=args.expected_canonical_sha256, + repository_head=args.repository_head, + ) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + print(render_sec_fundamentals_patch_preview(result)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_launchers.py b/tests/test_launchers.py index 8f988390..003cfaef 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -398,6 +398,57 @@ def test_sec_fundamentals_preview_is_explicit_capped_and_no_write(): ) +def test_sec_fundamentals_patch_preview_launcher_is_explicit_and_no_write(): + missing = subprocess.run( + ["make", "--dry-run", "sec-fundamentals-patch-preview"], + capture_output=True, + text=True, + check=False, + ) + assert missing.returncode != 0 + assert "SEC_PREVIEW is required" in missing.stderr + + missing_hashes = subprocess.run( + [ + "make", + "--dry-run", + "sec-fundamentals-patch-preview", + "SEC_PREVIEW=/tmp/reviewed-sec-preview.json", + ], + capture_output=True, + text=True, + check=False, + ) + assert missing_hashes.returncode != 0 + assert "EXPECTED_SEC_PREVIEW_SHA256 is required" in missing_hashes.stderr + + result = subprocess.run( + [ + "make", + "--dry-run", + "sec-fundamentals-patch-preview", + "SEC_PREVIEW=/tmp/reviewed-sec-preview.json", + f"EXPECTED_SEC_PREVIEW_SHA256={'1' * 64}", + f"EXPECTED_CANONICAL_SHA256={'2' * 64}", + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 + assert ( + "PYTHONDONTWRITEBYTECODE=1 python3 -m src.sec_fundamentals_patch_preview " + "--sec-preview-path \"/tmp/reviewed-sec-preview.json\" " + "--canonical-path \"data/fundamentals.csv\" " + f"--expected-sec-preview-sha256 \"{'1' * 64}\" " + f"--expected-canonical-sha256 \"{'2' * 64}\"" + in result.stdout + ) + assert re.search(r'--repository-head "[0-9a-f]{40}"', result.stdout) + for forbidden in ("--output", "apply", "readiness", "materialize", "provider"): + assert forbidden not in result.stdout.lower() + + def test_reviewed_batch_packet_targets_forward_one_named_profile(): makefile = Path("Makefile").read_text(encoding="utf-8") for target in ("reviewed-batch", "fundamentals-batch-proof", "peer-batch-proof"): diff --git a/tests/test_sec_fundamentals_patch_preview.py b/tests/test_sec_fundamentals_patch_preview.py new file mode 100644 index 00000000..ed86d4e5 --- /dev/null +++ b/tests/test_sec_fundamentals_patch_preview.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import csv +import hashlib +import io +import json + +import pytest + +from src.sec_fundamentals_patch_preview import ( + build_sec_fundamentals_patch_preview, + main, + render_sec_fundamentals_patch_preview, +) + + +_REPOSITORY_HEAD = "08fa35efb2759ca86785b1c2c95bc5cbfae4a9f4" + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _coherently_replace_aapl_cik_with_amd(packet: dict[str, object]) -> None: + row = packet["tickers"][0] + row["sec_cik"] = "0000002488" + source_url = "https://data.sec.gov/api/xbrl/companyfacts/CIK0000002488.json" + for field in row["fields"]: + field["sec_cik"] = "0000002488" + field["source_url"] = source_url + for ref in field["source_refs"]: + ref["source_url"] = source_url + + +def _field( + ticker: str, + field: str, + canonical_column: str, + canonical_value: object, + candidate_value: object, + *, + value_status: str = "changed", +) -> dict[str, object]: + cik = {"AAPL": "0000320193", "NVDA": "0001045810", "AMD": "0000002488"}[ticker] + accession = { + "AAPL": "0000320193-25-000079", + "NVDA": "0001045810-26-000021", + "AMD": "0000002488-26-000018", + }[ticker] + filing_date = {"AAPL": "2025-10-31", "NVDA": "2026-02-25", "AMD": "2026-02-04"}[ticker] + unit = "date" if field == "filing_dates" else "USD" + return { + "ticker": ticker, + "sec_cik": cik, + "field": field, + "canonical_column": canonical_column, + "canonical_value": canonical_value, + "candidate_value": candidate_value, + "classification": "approved_direct", + "commercial_rights_approved": True, + "field_scope_status": "approved", + "schema_status": "existing_canonical", + "publishability_blocker": "none", + "value_kind": "direct", + "value_status": value_status, + "unit": unit, + "source_units": [unit], + "source_rights_status": "approved", + "retrieval_timestamp": "2026-08-20T18:28:53Z", + "source_url": f"https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json", + "source_refs": [ + { + "accession": accession, + "concept": "RevenueFromContractWithCustomerExcludingAssessedTax", + "filed": filing_date, + "fiscal_period": "FY", + "fiscal_year": 2025, + "form": "10-K", + "period_end": "2025-12-27" if ticker == "AMD" else "2025-09-27", + "period_start": "2024-12-29" if ticker == "AMD" else "2024-09-29", + "retrieval_timestamp": "2026-08-20T18:28:53Z", + "source_url": f"https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json", + "taxonomy": "us-gaap", + "underlying_fact_unit": "USD" if field == "filing_dates" else None, + "unit": unit, + } + ], + } + + +def _sec_packet() -> bytes: + rows = [ + { + "ticker": "AAPL", + "sec_cik": "0000320193", + "future_apply_candidate_fields": ["revenue", "filing_dates"], + "fields": [ + _field("AAPL", "revenue", "revenue", 265_595_000_000, 416_161_000_000), + _field("AAPL", "filing_dates", "sec_filed_date", "2018-11-05", "2025-10-31"), + ], + }, + { + "ticker": "NVDA", + "sec_cik": "0001045810", + "future_apply_candidate_fields": [], + "fields": [ + _field( + "NVDA", + "revenue", + "revenue", + 215_938_000_000, + 215_938_000_000, + value_status="unchanged", + ), + _field( + "NVDA", + "filing_dates", + "sec_filed_date", + "2026-02-25", + "2026-02-25", + value_status="unchanged", + ), + ], + }, + { + "ticker": "AMD", + "sec_cik": "0000002488", + "future_apply_candidate_fields": ["revenue", "filing_dates"], + "fields": [ + _field("AMD", "revenue", "revenue", 5_329_000_000, 34_639_000_000), + _field("AMD", "filing_dates", "sec_filed_date", "2018-02-27", "2026-02-04"), + ], + }, + ] + packet = { + "status": "inspection_only", + "source": "sec_companyfacts", + "source_rights_mutated": False, + "canonical_apply_authorized": False, + "repository_writes": [], + "requested_tickers": ["AAPL", "NVDA", "AMD"], + "retrieval_timestamp": "2026-08-20T18:28:53Z", + "schema_delta": { + "staged_extra_columns": ["currency"], + "canonical_columns_not_produced": ["market_cap"], + "full_row_rewrite_risk": True, + }, + "tickers": rows, + } + return (json.dumps(packet, indent=2, sort_keys=True) + "\n").encode() + + +def _canonical() -> bytes: + return ( + "ticker,revenue,sec_filed_date,market_cap,source\n" + "AAPL,265595000000,2018-11-05,3000000000000,legacy\n" + "NVDA,215938000000,2026-02-25,4000000000000,reviewed\n" + "AMD,5329000000,2018-02-27,250000000000,legacy\n" + "MSFT,100,2025-01-01,999,untouched\n" + ).encode() + + +def _build(sec_packet: bytes | None = None, canonical: bytes | None = None, **kwargs): + sec_packet = sec_packet or _sec_packet() + canonical = canonical or _canonical() + return build_sec_fundamentals_patch_preview( + sec_packet, + canonical, + canonical_path="data/fundamentals.csv", + expected_sec_preview_sha256=_sha256(sec_packet), + expected_canonical_sha256=_sha256(canonical), + repository_head=_REPOSITORY_HEAD, + **kwargs, + ) + + +def test_patch_preview_selects_only_four_reviewed_changed_cells(): + result = _build() + + assert [(cell["ticker"], cell["field"]) for cell in result["patch_cells"]] == [ + ("AAPL", "revenue"), + ("AAPL", "filing_dates"), + ("AMD", "revenue"), + ("AMD", "filing_dates"), + ] + assert result["changed_cell_count"] == 4 + assert result["status"] == "inspection_only" + assert result["canonical_apply_authorized"] is False + assert result["repository_writes"] == [] + assert result["preconditions"]["repository_head"] == _REPOSITORY_HEAD + assert result["next_owner_decision"].startswith("Separately authorize or reject") + for cell in result["patch_cells"]: + assert cell["commercial_rights_approved"] is True + assert cell["source_rights_status"] == "approved" + assert cell["field_scope_status"] == "approved" + assert cell["schema_status"] == "existing_canonical" + + +def test_patch_preview_proves_schema_rows_order_and_unrelated_cells_unchanged(): + result = _build() + proof = result["in_memory_projection_proof"] + + assert proof == { + "column_count_before": 5, + "column_count_after": 5, + "schema_added_columns": [], + "schema_removed_columns": [], + "row_count_before": 4, + "row_count_after": 4, + "row_order_unchanged": True, + "full_row_replacement": False, + "staged_input_used": False, + "untouched_cells_unchanged": True, + "untouched_cells_sha256_before": proof["untouched_cells_sha256_after"], + "untouched_cells_sha256_after": proof["untouched_cells_sha256_after"], + "untouched_rows_unchanged": True, + "untouched_rows_sha256_before": proof["untouched_rows_sha256_after"], + "untouched_rows_sha256_after": proof["untouched_rows_sha256_after"], + "untouched_columns_unchanged": True, + "untouched_columns_sha256_before": proof["untouched_columns_sha256_after"], + "untouched_columns_sha256_after": proof["untouched_columns_sha256_after"], + "projected_semantic_matrix_sha256": proof["projected_semantic_matrix_sha256"], + } + + +def test_patch_preview_binds_canonical_and_sec_packet_bytes_and_is_deterministic(): + first = _build() + second = _build() + + assert len(first["preconditions"]["canonical_sha256"]) == 64 + assert len(first["preconditions"]["sec_preview_sha256"]) == 64 + assert len(first["projection_identity"]) == 64 + assert render_sec_fundamentals_patch_preview(first) == render_sec_fundamentals_patch_preview(second) + + semantically_equal_bytes = _sec_packet().replace(b"{\n", b"{ \n", 1) + with pytest.raises(ValueError, match="SEC preview hash precondition"): + build_sec_fundamentals_patch_preview( + semantically_equal_bytes, + _canonical(), + canonical_path="data/fundamentals.csv", + expected_sec_preview_sha256=_sha256(_sec_packet()), + expected_canonical_sha256=_sha256(_canonical()), + repository_head=_REPOSITORY_HEAD, + ) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda packet: packet.update(status="ready"), "inspection_only"), + (lambda packet: packet.update(canonical_apply_authorized=True), "apply authorization"), + ( + lambda packet: packet["tickers"][0]["future_apply_candidate_fields"].append("shares_outstanding"), + "unexpected candidate field", + ), + ( + lambda packet: packet["tickers"][0]["fields"][0].update(field_scope_status="review_required"), + "approved field scope", + ), + ( + lambda packet: packet["tickers"][0]["fields"][0]["source_refs"][0].update( + source_url="https://query1.finance.yahoo.com/x" + ), + "official SEC", + ), + ( + lambda packet: packet["tickers"][0]["fields"][0]["source_refs"][0].update( + source_url="https://data.sec.gov/api/xbrl/companyfacts/CIK0000002488.json" + ), + "reviewed SEC CIK", + ), + ( + lambda packet: packet["tickers"][0]["fields"][0]["source_refs"][0].update( + retrieval_timestamp="2026-08-20T18:28:54Z" + ), + "retrieval timestamp", + ), + ( + lambda packet: packet["tickers"][0]["fields"][0].update( + unit="shares", source_units=["shares"] + ), + "USD units", + ), + (_coherently_replace_aapl_cik_with_amd, "ticker-to-CIK"), + ], +) +def test_patch_preview_fails_closed_for_unreviewed_or_untrusted_evidence(mutation, message): + packet = json.loads(_sec_packet()) + mutation(packet) + + with pytest.raises(ValueError, match=message): + mutated = (json.dumps(packet, sort_keys=True) + "\n").encode() + _build(sec_packet=mutated) + + +def test_patch_preview_rejects_stale_canonical_preconditions(): + canonical = _canonical().replace(b"265595000000", b"265595000001") + + with pytest.raises(ValueError, match="canonical hash precondition"): + build_sec_fundamentals_patch_preview( + _sec_packet(), + canonical, + canonical_path="data/fundamentals.csv", + expected_sec_preview_sha256=_sha256(_sec_packet()), + expected_canonical_sha256=_sha256(_canonical()), + repository_head=_REPOSITORY_HEAD, + ) + + +def test_patch_preview_rejects_duplicate_ticker_rows_and_schema_expansion(): + packet = json.loads(_sec_packet()) + packet["tickers"].append(packet["tickers"][0]) + with pytest.raises(ValueError, match="duplicate ticker"): + _build(sec_packet=(json.dumps(packet, sort_keys=True) + "\n").encode()) + + canonical = _canonical().replace(b"source\n", b"source,currency\n") + with pytest.raises(ValueError, match="canonical row width"): + _build(canonical=canonical) + + +def test_patch_preview_does_not_write_files(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + result = _build() + + assert result["repository_writes"] == [] + assert list(tmp_path.iterdir()) == [] + + +def test_canonical_fixture_is_well_formed(): + rows = list(csv.DictReader(io.StringIO(_canonical().decode()))) + assert [row["ticker"] for row in rows] == ["AAPL", "NVDA", "AMD", "MSFT"] + + +def test_cli_reads_exact_inputs_and_writes_only_json_to_stdout(tmp_path, capsys): + sec_path = tmp_path / "sec-preview.json" + canonical_path = tmp_path / "fundamentals.csv" + sec_path.write_bytes(_sec_packet()) + canonical_path.write_bytes(_canonical()) + before = sorted(path.name for path in tmp_path.iterdir()) + + assert main( + [ + "--sec-preview-path", + str(sec_path), + "--canonical-path", + str(canonical_path), + "--expected-sec-preview-sha256", + _sha256(_sec_packet()), + "--expected-canonical-sha256", + _sha256(_canonical()), + "--repository-head", + _REPOSITORY_HEAD, + ] + ) == 0 + + output = json.loads(capsys.readouterr().out) + assert output["changed_cell_count"] == 4 + assert output["preconditions"]["canonical_path"] == str(canonical_path) + assert sorted(path.name for path in tmp_path.iterdir()) == before