From e76f5093de3110cae0dee07c2475f23b5adb6edb Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sat, 15 Aug 2026 15:43:46 -0400 Subject: [PATCH 1/5] Reveal existing screener context --- src/dashboard.py | 132 ++++++++++++++++++++++++++------ src/stock_report.py | 18 ++++- tests/test_dashboard_helpers.py | 41 ++++++++++ tests/test_stock_report.py | 19 +++++ 4 files changed, 185 insertions(+), 25 deletions(-) diff --git a/src/dashboard.py b/src/dashboard.py index 2c4958d2..4b7cffec 100644 --- a/src/dashboard.py +++ b/src/dashboard.py @@ -11487,24 +11487,51 @@ def _atr_or_volatility_source_label(source: object) -> str: return "Volatility source unavailable" +def _screener_context_value(values: Mapping[str, object], field: str) -> object: + """Read serialized screener fields without assuming one key casing.""" + + if field in values: + return values.get(field) + normalized = field.casefold() + return next( + (value for key, value in values.items() if str(key).casefold() == normalized), + None, + ) + + def stock_report_technical_context_cards(report_payload: dict[str, object]) -> list[dict[str, object]]: screener_context = report_payload.get("screener_context", {}) or {} momentum = screener_context.get("momentum_leaders", {}) or {} watchlist = screener_context.get("final_watchlist", {}) or {} - setup_status = format_missing(momentum.get("SetupStatus") or watchlist.get("SetupStatus"), "Not available") - final_state = format_missing(watchlist.get("FinalState"), "Not available") - rs_percentile = momentum.get("RSPercentile") - relative_spy = momentum.get("RelativeReturnVsSPY") - relative_qqq = momentum.get("RelativeReturnVsQQQ") - volume_ratio = momentum.get("VolumeRatio") + setup_status = format_missing( + _screener_context_value(momentum, "SetupStatus") + or _screener_context_value(watchlist, "SetupStatus"), + "Not available", + ) + final_state = format_missing( + _screener_context_value(watchlist, "FinalState"), + "Not available", + ) + rs_percentile = _screener_context_value(momentum, "RSPercentile") + relative_spy = _screener_context_value(momentum, "RelativeReturnVsSPY") + relative_qqq = _screener_context_value(momentum, "RelativeReturnVsQQQ") + volume_ratio = _screener_context_value(momentum, "VolumeRatio") volume_ratio_display = report_display_value(volume_ratio, "number") volume_title = f"Volume {volume_ratio_display}x" if volume_ratio_display != "Not available" else "Volume ratio not available" - volatility_proxy = momentum.get("ATRorVolatilityPct") - volatility_source = _atr_or_volatility_source_label(momentum.get("ATRorVolatilitySource")) + volatility_proxy = _screener_context_value(momentum, "ATRorVolatilityPct") + volatility_source = _atr_or_volatility_source_label( + _screener_context_value(momentum, "ATRorVolatilitySource") + ) ma_stack = [ - _technical_distance_label(momentum.get("DistanceFrom10EMA"), "10 EMA"), - _technical_distance_label(momentum.get("DistanceFrom21EMA"), "21 EMA"), - _technical_distance_label(momentum.get("DistanceFrom50SMA"), "50 SMA"), + _technical_distance_label( + _screener_context_value(momentum, "DistanceFrom10EMA"), "10 EMA" + ), + _technical_distance_label( + _screener_context_value(momentum, "DistanceFrom21EMA"), "21 EMA" + ), + _technical_distance_label( + _screener_context_value(momentum, "DistanceFrom50SMA"), "50 SMA" + ), ] return [ { @@ -11681,18 +11708,77 @@ def stock_report_technical_context_frame(report_payload: dict[str, object]) -> p momentum = screener_context.get("momentum_leaders", {}) or {} watchlist = screener_context.get("final_watchlist", {}) or {} rows = [ - {"Metric": "Setup Status", "Value": format_missing(momentum.get("SetupStatus") or watchlist.get("SetupStatus"))}, - {"Metric": "Final State", "Value": format_missing(watchlist.get("FinalState"))}, - {"Metric": "RS Percentile", "Value": report_display_value(momentum.get("RSPercentile"), "number")}, - {"Metric": "Relative Return vs SPY", "Value": report_display_value(momentum.get("RelativeReturnVsSPY"), "percent")}, - {"Metric": "Relative Return vs QQQ", "Value": report_display_value(momentum.get("RelativeReturnVsQQQ"), "percent")}, - {"Metric": "10 EMA Distance", "Value": report_display_value(momentum.get("DistanceFrom10EMA"), "percent")}, - {"Metric": "21 EMA Distance", "Value": report_display_value(momentum.get("DistanceFrom21EMA"), "percent")}, - {"Metric": "50 SMA Distance", "Value": report_display_value(momentum.get("DistanceFrom50SMA"), "percent")}, - {"Metric": "Average Volume 20D", "Value": report_display_value(momentum.get("AvgVolume20D"), "integer")}, - {"Metric": "Volume Ratio", "Value": report_display_value(momentum.get("VolumeRatio"), "number")}, - {"Metric": "ATR / Volatility Proxy", "Value": report_display_value(momentum.get("ATRorVolatilityPct"), "percent")}, - {"Metric": "Volatility Source", "Value": _atr_or_volatility_source_label(momentum.get("ATRorVolatilitySource"))}, + { + "Metric": "Setup Status", + "Value": format_missing( + _screener_context_value(momentum, "SetupStatus") + or _screener_context_value(watchlist, "SetupStatus") + ), + }, + { + "Metric": "Final State", + "Value": format_missing(_screener_context_value(watchlist, "FinalState")), + }, + { + "Metric": "RS Percentile", + "Value": report_display_value( + _screener_context_value(momentum, "RSPercentile"), "number" + ), + }, + { + "Metric": "Relative Return vs SPY", + "Value": report_display_value( + _screener_context_value(momentum, "RelativeReturnVsSPY"), "percent" + ), + }, + { + "Metric": "Relative Return vs QQQ", + "Value": report_display_value( + _screener_context_value(momentum, "RelativeReturnVsQQQ"), "percent" + ), + }, + { + "Metric": "10 EMA Distance", + "Value": report_display_value( + _screener_context_value(momentum, "DistanceFrom10EMA"), "percent" + ), + }, + { + "Metric": "21 EMA Distance", + "Value": report_display_value( + _screener_context_value(momentum, "DistanceFrom21EMA"), "percent" + ), + }, + { + "Metric": "50 SMA Distance", + "Value": report_display_value( + _screener_context_value(momentum, "DistanceFrom50SMA"), "percent" + ), + }, + { + "Metric": "Average Volume 20D", + "Value": report_display_value( + _screener_context_value(momentum, "AvgVolume20D"), "integer" + ), + }, + { + "Metric": "Volume Ratio", + "Value": report_display_value( + _screener_context_value(momentum, "VolumeRatio"), "number" + ), + }, + { + "Metric": "ATR / Volatility Proxy", + "Value": report_display_value( + _screener_context_value(momentum, "ATRorVolatilityPct"), "percent" + ), + }, + { + "Metric": "Volatility Source", + "Value": _atr_or_volatility_source_label( + _screener_context_value(momentum, "ATRorVolatilitySource") + ), + }, ] return pd.DataFrame(rows) diff --git a/src/stock_report.py b/src/stock_report.py index 8c462581..bf24991d 100644 --- a/src/stock_report.py +++ b/src/stock_report.py @@ -879,12 +879,26 @@ def _atr_or_volatility_source_label(source: Any) -> str: return "Volatility source unavailable" +def _screener_context_value(values: dict[str, Any], field: str) -> Any: + """Read serialized screener fields without assuming one key casing.""" + + if field in values: + return values.get(field) + normalized = field.casefold() + return next( + (value for key, value in values.items() if str(key).casefold() == normalized), + None, + ) + + def _stock_report_volatility_lines(payload: dict[str, Any]) -> list[str]: momentum = ((payload.get("screener_context") or {}).get("momentum_leaders") or {}) - volatility_value = momentum.get("ATRorVolatilityPct") + volatility_value = _screener_context_value(momentum, "ATRorVolatilityPct") if _display_value(volatility_value) == "Not available": return ["- ATR / volatility: Not available; missing values stay visible instead of guessed."] - source_label = _atr_or_volatility_source_label(momentum.get("ATRorVolatilitySource")) + source_label = _atr_or_volatility_source_label( + _screener_context_value(momentum, "ATRorVolatilitySource") + ) if source_label == "Volatility proxy approximation": suffix = " This is an approximation from close-to-close volatility because high/low ATR inputs were unavailable." elif source_label == "Volatility source unavailable": diff --git a/tests/test_dashboard_helpers.py b/tests/test_dashboard_helpers.py index feaa45dd..b70258fb 100644 --- a/tests/test_dashboard_helpers.py +++ b/tests/test_dashboard_helpers.py @@ -18866,6 +18866,47 @@ def test_stock_report_technical_context_cards_do_not_append_units_to_missing_vol assert "nan" not in rendered +def test_stock_report_technical_context_reads_real_lowercase_screener_payload(): + payload = { + "screener_context": { + "momentum_leaders": { + "setupstatus": "Setup Forming", + "rspercentile": 87, + "relativereturnvsspy": 0.12, + "relativereturnvsqqq": 0.08, + "distancefrom10ema": 0.0267, + "distancefrom21ema": 0.0572, + "distancefrom50sma": 0.139, + "avgvolume20d": 48_364_661, + "volumeratio": 0.68, + "atrorvolatilitypct": 0.0171, + }, + "final_watchlist": { + "finalstate": "Setup Forming", + "setupstatus": "Setup Forming", + }, + } + } + + cards = dashboard.stock_report_technical_context_cards(payload) + rendered = " ".join(str(value) for card in cards for value in card.values()).lower() + frame = dashboard.stock_report_technical_context_frame(payload).set_index("Metric")["Value"] + + assert "setup forming" in rendered + assert "vs spy 12.0%" in rendered + assert "above 10 ema" in rendered + assert "volume 0.68x" in rendered + assert "1.7%" in rendered + assert frame["Setup Status"] == "Setup Forming" + assert frame["Final State"] == "Setup Forming" + assert frame["10 EMA Distance"] == "2.7%" + assert frame["21 EMA Distance"] == "5.7%" + assert frame["50 SMA Distance"] == "13.9%" + assert frame["Average Volume 20D"] == "48,364,661" + assert frame["Volume Ratio"] == "0.68" + assert frame["ATR / Volatility Proxy"] == "1.7%" + + def test_stock_report_technical_context_frame_formats_missing_values_cleanly(): frame = dashboard.stock_report_technical_context_frame({"screener_context": {}}) diff --git a/tests/test_stock_report.py b/tests/test_stock_report.py index 46e888d9..ea454f06 100644 --- a/tests/test_stock_report.py +++ b/tests/test_stock_report.py @@ -33,6 +33,7 @@ _stock_report_missing_data_lines, _stock_report_reader_guide_lines, _stock_report_reader_question_lines, + _stock_report_volatility_lines, _stock_report_valuation_lines, _stock_report_purpose_fields, build_readiness_only_markdown, @@ -46,6 +47,24 @@ RICH_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "rich_local_data" +def test_stock_report_volatility_lines_read_real_lowercase_screener_payload(): + lines = _stock_report_volatility_lines( + { + "screener_context": { + "momentum_leaders": { + "atrorvolatilitypct": 0.0171, + "atrorvolatilitysource": "volatility_proxy", + } + } + } + ) + + assert lines == [ + "- ATR / volatility: 1.7% (Volatility proxy approximation). " + "This is an approximation from close-to-close volatility because high/low ATR inputs were unavailable." + ] + + def test_stock_report_formats_bare_make_commands_as_copyable_inline_commands(): text = _format_inline_make_commands( "Run make focus-fundamentals TICKER=META, then make imports-validate before review. " From f53bca011765b180b52c9697cef0f4811fc91b50 Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sat, 15 Aug 2026 15:47:08 -0400 Subject: [PATCH 2/5] Design no-write SEC fundamentals preview --- .../2026-08-15-sec-fundamentals-preview.md | 66 +++++++++++++++++ ...6-08-15-sec-fundamentals-preview-design.md | 71 +++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-15-sec-fundamentals-preview.md create mode 100644 docs/superpowers/specs/2026-08-15-sec-fundamentals-preview-design.md diff --git a/docs/superpowers/plans/2026-08-15-sec-fundamentals-preview.md b/docs/superpowers/plans/2026-08-15-sec-fundamentals-preview.md new file mode 100644 index 00000000..01834df3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-sec-fundamentals-preview.md @@ -0,0 +1,66 @@ +# SEC Fundamentals No-Write Preview Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:test-driven-development for each behavior change. Track steps with checkbox (`- [ ]`) syntax. + +**Goal:** Build a capped official-SEC comparison that exposes coherent annual actual candidates and their blockers without modifying canonical, staged, cached, readiness, or generated data. + +**Architecture:** Extend the existing SEC adapter with true no-cache reads and private per-field provenance. Add a pure comparison module that reads canonical/staged headers, fetches only official SEC JSON in memory, classifies field coherence and rights, and prints deterministic JSON. Expose it through one Make target. + +**Tech Stack:** Python 3.12, urllib, pandas, YAML-backed source-rights registry, argparse, JSON, pytest, Make. + +## Global Constraints + +- Maximum five explicit tickers; first audited cohort AAPL, AMZN, GOOG. +- Official SEC ticker-map and Companyfacts endpoints only. +- No provider fallback, canonical apply, import staging, cache write, readiness mutation, or rights change. +- Missing facts remain unavailable; derived values are labelled derived and blocked pending exact field-scope review. +- Stage only explicit source/docs/test/Make paths. Never stage `data/`, `outputs/`, caches, or imports. + +### Task 1: True No-Cache SEC Adapter + +**Files:** +- Modify: `src/providers/sec_companyfacts.py` +- Modify: `tests/test_sec_companyfacts.py` + +- [ ] Write RED tests proving ticker-map and Companyfacts no-cache calls make the expected official request but create no cache directory or file. +- [ ] Add a `cache=False` ticker-map path and avoid resolving a Companyfacts cache path when cache is disabled. +- [ ] Preserve existing cached staging behavior and rerun the full SEC provider test file. + +### Task 2: Provenance-Aware Candidate Extraction + +**Files:** +- Modify: `src/providers/sec_companyfacts.py` +- Modify: `tests/test_sec_companyfacts.py` + +- [ ] Write RED tests for direct record metadata and derived component provenance. +- [ ] Add private `_field_provenance` output to the extractor; confirm staging rows still omit private keys. +- [ ] Prove missing facts stay `None` and no derived field is described as directly reported. + +### Task 3: Pure Preview Comparison + +**Files:** +- Create: `src/sec_fundamentals_preview.py` +- Create: `tests/test_sec_fundamentals_preview.py` + +- [ ] Write RED tests for explicit input, five-ticker cap, deterministic field deltas, classification precedence, period/accession coherence, malformed payloads, staged schema deltas, AAPL mixed-period visibility, and GOOG missing shares. +- [ ] Implement input parsing, official fetch orchestration, canonical/staged read-only projection, field comparison, coherence checks, rights review, and deterministic JSON rendering. +- [ ] Keep one ticker's failure isolated and report it without fabricated values. + +### Task 4: Command Surface + +**Files:** +- Modify: `Makefile` +- Modify: `tests/test_launchers.py` + +- [ ] Write RED tests for help text and exact command wiring. +- [ ] Add `make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG`; require `TICKERS` and pass no output/cache/apply argument. +- [ ] Prove the target cannot invoke stage, apply, readiness, or fallback-provider commands. + +### Task 5: Cohort Audit and Verification + +- [ ] Run focused provider, preview, launcher, stock-report, and data-quality tests. +- [ ] Run targeted Ruff/compile checks and `git diff --check`. +- [ ] Compare all tracked `data/`/`outputs/` hashes and both ignored SEC-state hashes to their Stage 0 manifests. +- [ ] Run the live AAPL/AMZN/GOOG preview once with the configured SEC user agent and save any durable evidence only under a fresh `/tmp` directory. +- [ ] Self-review the complete diff for Critical/Important issues and resolve them before local commits. +- [ ] Commit only named source/docs/tests/Make paths. Do not push. diff --git a/docs/superpowers/specs/2026-08-15-sec-fundamentals-preview-design.md b/docs/superpowers/specs/2026-08-15-sec-fundamentals-preview-design.md new file mode 100644 index 00000000..083b0342 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-sec-fundamentals-preview-design.md @@ -0,0 +1,71 @@ +# SEC Fundamentals No-Write Preview Design + +**Status:** Approved for bounded local implementation. Canonical apply, source-rights expansion, release recording, and remote synchronization are not authorized. + +## Purpose + +Provide a deterministic, inspection-only comparison between the current canonical fundamentals row and a fresh official SEC Companyfacts candidate for at most five explicitly named tickers. The first cohort is AAPL, AMZN, and GOOG. + +The preview exists to expose stale, mixed-period, missing, unsupported, and derived evidence. It does not promote readiness or decide that a candidate may be published. + +## Request and No-Write Boundary + +- Accept only explicit tickers and reject an empty list or more than five unique tickers. +- Fetch only the SEC ticker map and SEC Companyfacts HTTPS endpoints with an identifying `SEC_USER_AGENT`. +- Do not call any fallback provider, scraper, search result, or paid API. +- Add a true no-cache adapter path: when cache is disabled, neither ticker-map nor Companyfacts path resolution may create directories or files. +- Read canonical `data/fundamentals.csv` and, when present, the ignored staged fundamentals header only for comparison. Never normalize, rewrite, delete, or apply either file. +- Print deterministic JSON to stdout. Tests may use an isolated `/tmp` fixture, but production code has no output-file option. + +## Candidate and Provenance Model + +The existing SEC extractor remains the source of candidate calculations. It will also expose private field-provenance metadata that the staging writer omits. Each field reports: + +- canonical and SEC candidate values; +- `changed`, `unchanged`, or `missing` value status; +- fiscal period start/end when applicable; +- filing date, accession, form, taxonomy, concept, unit, and exact SEC Companyfacts URL; +- `direct` or `derived` value kind; +- one fail-closed classification and its publishability blocker. + +The preview covers the SEC-backed canonical fields currently produced by the extractor: revenue, revenue growth, EPS, free cash flow, FCF margin, profit margin, operating margin, EBITDA, cash, debt, and shares outstanding. Source components such as operating income, cash from operations, and capital expenditures remain provenance inputs, not silently added canonical columns. + +## Coherence Rules + +Revenue's latest annual record defines the candidate fiscal-period anchor. Annual flow facts must use that period. Instant facts must be tied to the same filing accession or exact period end. Derived fields inherit every component's context. Revenue growth may intentionally use the anchored annual period and its immediately prior annual period. + +When a component cannot be tied to that context, the observed value may be shown for diagnosis but is classified `period_conflict` or `source_context_ambiguous` and is blocked from publication. Missing facts remain unavailable; they never become zero. + +## Source-Rights Classification + +Classifications are resolved in this order: + +1. `missing` when no candidate fact exists. +2. `period_conflict` when a period-specific fact conflicts with the annual anchor. +3. `source_context_ambiguous` when filing/accession context cannot be tied to the anchor. +4. `derived_scope_review_required` for every calculated field, even when its source components are registered. +5. `approved_direct` only when the exact direct field is listed for `sec_companyfacts` in `config/source_rights.yml`. +6. `unsupported` for a present direct field outside the registered field scope. + +No preview result changes the source-rights registry. Filing metadata is evidence context, not permission to publish an otherwise unsupported field. + +## Schema Risk + +The result reports: + +- candidate provenance components that are not canonical columns; +- canonical columns not produced by this SEC candidate; +- columns found in the ignored staged fundamentals file but absent from canonical fundamentals; +- columns that a naïve full-row rewrite would drop or add. + +The preview never adds `currency` or any other column and never rewrites the full dataset for one ticker. + +## Failure States + +- Missing `SEC_USER_AGENT`, invalid ticker input, a non-SEC request, malformed payload, unresolved CIK, or missing canonical row is reported explicitly and fails closed. +- One ticker's failure does not fabricate values for that ticker or change another ticker's result. +- Analyst estimates, targets, ratings, recommendations, prices, peer decisions, and quarterly cash-flow derivations are outside this command. + +## Verification + +Tests must prove the ticker cap, official-endpoint restriction, true no-cache behavior, deterministic deltas, direct-versus-derived classification, missing and malformed fail-closed behavior, mixed-period blocking, staged schema reporting, AAPL mixed-period visibility, and GOOG explicit-share unavailability. Before and after hashes must prove that tracked `data/` and `outputs/` plus the existing ignored cache/staging files remain byte-identical. From 7bac024d214284822fb07d40273759d06f125097 Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sat, 15 Aug 2026 16:28:28 -0400 Subject: [PATCH 3/5] Add no-write SEC fundamentals preview --- Makefile | 8 + src/providers/sec_companyfacts.py | 111 +++- src/sec_fundamentals_preview.py | 670 +++++++++++++++++++++++++ tests/test_launchers.py | 30 ++ tests/test_sec_companyfacts.py | 129 +++++ tests/test_sec_fundamentals_preview.py | 518 +++++++++++++++++++ 6 files changed, 1453 insertions(+), 13 deletions(-) create mode 100644 src/sec_fundamentals_preview.py create mode 100644 tests/test_sec_fundamentals_preview.py diff --git a/Makefile b/Makefile index 02da118a..f02f9b30 100644 --- a/Makefile +++ b/Makefile @@ -356,6 +356,7 @@ help-full: @echo "Preview-first fundamentals and universe imports:" @echo " export SEC_USER_AGENT='Name email@example.com'" @echo " make sec-stage TICKERS=NVDA,MSFT" + @echo " make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" @echo " make yfinance-stage TICKERS=NVDA" @echo " make fundamentals-source-ladder TICKERS=NVDA" @echo " Try SEC, yfinance, FMP, Alpha Vantage, then Finnhub before stopping at reviewed blocker evidence" @@ -1271,6 +1272,13 @@ endif --primary-document "$(or $(PRIMARY_DOCUMENT),nvda-20260426.htm)" \ --as-of "$(AS_OF)" +.PHONY: sec-fundamentals-preview +sec-fundamentals-preview: +ifndef TICKERS + $(error TICKERS is required, for example: make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG) +endif + @PYTHONDONTWRITEBYTECODE=1 python3 -m src.sec_fundamentals_preview --tickers "$(TICKERS)" + demo-dashboard-render-smoke: @STOCK_RESEARCH_DATA_PROFILE=demo python3 -m src.dashboard_render_smoke diff --git a/src/providers/sec_companyfacts.py b/src/providers/sec_companyfacts.py index f61256a3..b24fb60d 100644 --- a/src/providers/sec_companyfacts.py +++ b/src/providers/sec_companyfacts.py @@ -103,18 +103,27 @@ def load_sec_ticker_map( refresh: bool = False, sleep_seconds: float = 0.2, fetcher: Callable[[str, str, float], Any] | None = None, + cache: bool = True, ) -> dict[str, dict[str, Any]]: - cache_path = _cache_path(Path(cache_dir), "company_tickers.json") - if cache_path.exists() and not refresh: - payload = _read_json(cache_path) - if fetcher is None and _payload_row_count(payload) < MIN_SEC_TICKER_MAP_ROWS: + if not cache: + resolved_user_agent = _require_user_agent(user_agent) + payload = (fetcher or _fetch_json)( + SEC_TICKER_MAP_URL, + resolved_user_agent, + sleep_seconds, + ) + else: + cache_path = _cache_path(Path(cache_dir), "company_tickers.json") + if cache_path.exists() and not refresh: + payload = _read_json(cache_path) + if fetcher is None and _payload_row_count(payload) < MIN_SEC_TICKER_MAP_ROWS: + resolved_user_agent = _require_user_agent(user_agent) + payload = _fetch_json(SEC_TICKER_MAP_URL, resolved_user_agent, sleep_seconds) + _write_json(cache_path, payload) + else: resolved_user_agent = _require_user_agent(user_agent) - payload = _fetch_json(SEC_TICKER_MAP_URL, resolved_user_agent, sleep_seconds) + payload = (fetcher or _fetch_json)(SEC_TICKER_MAP_URL, resolved_user_agent, sleep_seconds) _write_json(cache_path, payload) - else: - resolved_user_agent = _require_user_agent(user_agent) - payload = (fetcher or _fetch_json)(SEC_TICKER_MAP_URL, resolved_user_agent, sleep_seconds) - _write_json(cache_path, payload) rows: Iterable[dict[str, Any]] if isinstance(payload, dict) and all(isinstance(value, dict) for value in payload.values()): @@ -177,9 +186,10 @@ def fetch_companyfacts( ) -> dict[str, Any]: resolved_user_agent = _require_user_agent(user_agent) normalized_cik = str(cik).zfill(10) - cache_root = Path(cache_dir) - cache_path = _companyfacts_cache_path(cache_root, normalized_cik) - if cache and cache_path.exists() and not refresh: + cache_path = ( + _companyfacts_cache_path(Path(cache_dir), normalized_cik) if cache else None + ) + if cache_path is not None and cache_path.exists() and not refresh: if fetcher is not None or cache_path.stat().st_size >= MIN_SEC_COMPANYFACTS_CACHE_BYTES: return _read_json(cache_path) payload = (fetcher or _fetch_json)( @@ -187,7 +197,7 @@ def fetch_companyfacts( resolved_user_agent, sleep_seconds, ) - if cache: + if cache_path is not None: _write_json(cache_path, payload) return payload @@ -311,6 +321,33 @@ def _metadata_from_record(record: SecFactRecord | None) -> dict[str, str | None] } +def _record_provenance(record: SecFactRecord) -> dict[str, Any]: + return { + "taxonomy": record.taxonomy, + "concept": record.concept, + "unit": record.unit, + "period_start": record.start, + "period_end": record.end, + "filed": record.filed, + "form": record.form, + "accession": record.accession, + "fiscal_year": record.fy, + "fiscal_period": record.fp, + } + + +def _field_provenance( + value_kind: str, + records: Iterable[SecFactRecord | None], +) -> dict[str, Any]: + return { + "value_kind": value_kind, + "records": [ + _record_provenance(record) for record in records if record is not None + ], + } + + def extract_fundamentals_from_companyfacts(companyfacts_json: dict[str, Any]) -> dict[str, Any]: warnings: list[str] = [] revenue_concepts = [ @@ -363,11 +400,13 @@ def extract_fundamentals_from_companyfacts(companyfacts_json: dict[str, Any]) -> ebitda_record = _latest_record(companyfacts_json, ebitda_concepts, annual_only=True) debt_value, debt_records = _sum_latest_records(companyfacts_json, debt_component_groups) + debt_value_kind = "derived" if debt_value is None: total_debt_record = _latest_record(companyfacts_json, total_debt_concepts, annual_only=False) if total_debt_record is not None: debt_value = _numeric_record_value(total_debt_record) debt_records = [total_debt_record] + debt_value_kind = "direct" revenue = _numeric_record_value(revenue_record) net_income = _numeric_record_value(net_income_record) @@ -442,6 +481,52 @@ def extract_fundamentals_from_companyfacts(companyfacts_json: dict[str, Any]) -> "sec_fact_warnings": " | ".join(sorted(set(warnings))) if warnings else None, "sec_entity_name": companyfacts_json.get("entityName"), "_warnings": sorted(set(warnings)), + "_field_provenance": { + "revenue": _field_provenance("direct", [revenue_record]), + "revenue_growth": _field_provenance( + "derived", revenue_series[:2] + ), + "eps": _field_provenance("direct", [eps_record]), + "free_cash_flow": _field_provenance( + "derived", [ocf_record, capex_record] + ), + "fcf_margin": _field_provenance( + "derived", [revenue_record, ocf_record, capex_record] + ), + "profit_margin": _field_provenance( + "derived", [revenue_record, net_income_record] + ), + "operating_margin": _field_provenance( + "derived", [revenue_record, operating_income_record] + ), + "ebitda": _field_provenance("direct", [ebitda_record]), + "cash": _field_provenance("direct", [cash_record]), + "debt": _field_provenance( + debt_value_kind, + debt_records, + ), + "shares_outstanding": _field_provenance( + "direct", [shares_record] + ), + }, + "_source_components": { + "net_income": { + "value": net_income, + **_field_provenance("direct", [net_income_record]), + }, + "cash_from_operations": { + "value": operating_cash_flow, + **_field_provenance("direct", [ocf_record]), + }, + "capital_expenditures": { + "value": capex, + **_field_provenance("direct", [capex_record]), + }, + "operating_income": { + "value": operating_income, + **_field_provenance("direct", [operating_income_record]), + }, + }, } return row diff --git a/src/sec_fundamentals_preview.py b/src/sec_fundamentals_preview.py new file mode 100644 index 00000000..c94d2084 --- /dev/null +++ b/src/sec_fundamentals_preview.py @@ -0,0 +1,670 @@ +"""Official-SEC, no-write annual fundamentals comparison.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping + +import pandas as pd + +from src.commercial_source_rights import ( + DEFAULT_REGISTRY_PATH, + load_source_rights_registry, + review_commercial_field_scope, +) +from src.providers.sec_companyfacts import ( + SEC_ANNUAL_FORMS, + SEC_COMPANYFACTS_URL, + extract_fundamentals_from_companyfacts, + fetch_companyfacts, + load_sec_ticker_map, + resolve_ticker_to_cik, +) + + +MAX_PREVIEW_TICKERS = 5 +PREVIEW_FIELDS = ( + "revenue", + "revenue_growth", + "eps", + "free_cash_flow", + "fcf_margin", + "profit_margin", + "operating_margin", + "ebitda", + "cash", + "debt", + "shares_outstanding", +) +DIRECT_RIGHTS_FIELDS = { + "revenue": "revenue", + "shares_outstanding": "shares_outstanding", +} +CANDIDATE_COMPONENT_FIELDS = ( + "net_income", + "cash_from_operations", + "capital_expenditures", + "operating_income", +) +_TICKER_PATTERN = re.compile(r"^[A-Z0-9][A-Z0-9.-]*$") + + +def parse_preview_tickers(value: str | Iterable[str]) -> list[str]: + raw_values = value.split(",") if isinstance(value, str) else list(value) + tickers: list[str] = [] + for raw in raw_values: + ticker = str(raw or "").strip().upper() + if not ticker: + continue + if not _TICKER_PATTERN.fullmatch(ticker): + raise ValueError(f"invalid ticker: {ticker!r}") + if ticker not in tickers: + tickers.append(ticker) + if not tickers: + raise ValueError("explicit ticker input is required") + if len(tickers) > MAX_PREVIEW_TICKERS: + raise ValueError("SEC fundamentals preview accepts at most five unique tickers") + return tickers + + +def _json_value(value: Any) -> Any: + if value is None: + return None + try: + if pd.isna(value): + return None + except (TypeError, ValueError): + pass + if hasattr(value, "item"): + value = value.item() + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + +def _values_equal(left: Any, right: Any) -> bool: + left = _json_value(left) + right = _json_value(right) + if left is None or right is None: + return left is right + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + return math.isclose(float(left), float(right), rel_tol=1e-12, abs_tol=0.0) + return left == right + + +def _read_canonical(path: Path) -> tuple[pd.DataFrame, list[str]]: + if not path.is_file(): + raise ValueError(f"canonical fundamentals file is unavailable: {path}") + frame = pd.read_csv(path) + if "ticker" not in frame.columns: + raise ValueError("canonical fundamentals file requires a ticker column") + frame = frame.copy() + frame["ticker"] = frame["ticker"].astype("string").str.upper().str.strip() + return frame.set_index("ticker", drop=False), list(frame.columns) + + +def _read_header(path: Path) -> list[str]: + if not path.is_file(): + return [] + return list(pd.read_csv(path, nrows=0).columns) + + +def _valid_companyfacts_payload(payload: Any) -> bool: + if not isinstance(payload, Mapping): + return False + facts = payload.get("facts") + if not isinstance(facts, Mapping): + return False + for taxonomy in facts.values(): + if not isinstance(taxonomy, Mapping): + return False + for fact in taxonomy.values(): + if not isinstance(fact, Mapping): + return False + units = fact.get("units") + if not isinstance(units, Mapping): + return False + for items in units.values(): + if not isinstance(items, list) or not all( + isinstance(item, Mapping) for item in items + ): + return False + return True + + +def _valid_annual_anchor(record: Mapping[str, Any]) -> bool: + return bool( + record.get("period_start") + and record.get("period_end") + and record.get("filed") + and record.get("accession") + and record.get("form") in SEC_ANNUAL_FORMS + and _has_annual_duration(record) + ) + + +def _has_annual_duration(record: Mapping[str, Any]) -> bool: + period_start = pd.to_datetime(record.get("period_start"), errors="coerce") + period_end = pd.to_datetime(record.get("period_end"), errors="coerce") + if pd.isna(period_start) or pd.isna(period_end) or period_start >= period_end: + return False + return 300 <= (period_end - period_start).days <= 430 + + +def _field_context( + field: str, + provenance: Mapping[str, Any], + *, + anchor_period_start: str | None, + anchor_period_end: str | None, + anchor_accession: str | None, +) -> tuple[str | None, str | None]: + records = provenance.get("records") + if not isinstance(records, list) or not records: + return None, None + normalized = [record for record in records if isinstance(record, Mapping)] + if not normalized: + return None, "source context is unavailable" + + for record in normalized: + if record.get("period_start") and record.get("form") not in SEC_ANNUAL_FORMS: + return "period_conflict", ( + f"flow fact form {record.get('form') or 'unavailable'} is not an annual filing" + ) + if record.get("period_start") and not _has_annual_duration(record): + return "period_conflict", ( + "Flow fact does not have a valid annual duration and date order." + ) + + if field == "revenue_growth": + if len(normalized) != 2: + return "source_context_ambiguous", ( + "Revenue growth requires exactly two complete annual records." + ) + if any( + not record.get(key) + for record in normalized + for key in ("period_end", "filed", "accession", "fiscal_year") + ): + return "source_context_ambiguous", ( + "Revenue growth requires two complete annual filing contexts." + ) + latest_start = normalized[0].get("period_start") + latest_end = normalized[0].get("period_end") + if ( + not anchor_period_start + or not anchor_period_end + or latest_start != anchor_period_start + or latest_end != anchor_period_end + ): + return "period_conflict", ( + "latest revenue-growth component does not match annual anchor " + f"{anchor_period_start or 'unavailable'} to {anchor_period_end or 'unavailable'}" + ) + latest_period_end = pd.to_datetime( + normalized[0]["period_end"], errors="coerce" + ) + prior_period_end = pd.to_datetime( + normalized[1]["period_end"], errors="coerce" + ) + if pd.isna(latest_period_end) or pd.isna(prior_period_end): + return "source_context_ambiguous", ( + "Revenue growth period-end context is unavailable." + ) + period_gap_days = (latest_period_end - prior_period_end).days + if not 300 <= period_gap_days <= 430: + return "period_conflict", ( + "Revenue growth requires the immediately adjacent prior annual period." + ) + return None, None + + for record in normalized: + period_end = record.get("period_end") + accession = record.get("accession") + period_start = record.get("period_start") + if period_start: + if not anchor_period_start or period_start != anchor_period_start: + return "period_conflict", ( + f"field period start {period_start or 'unavailable'} does not match annual anchor " + f"{anchor_period_start or 'unavailable'}" + ) + if not anchor_period_end or period_end != anchor_period_end: + return "period_conflict", ( + f"field period {period_end or 'unavailable'} does not match annual anchor {anchor_period_end or 'unavailable'}" + ) + elif not ( + anchor_period_end + and period_end == anchor_period_end + or anchor_accession + and accession == anchor_accession + ): + return "source_context_ambiguous", ( + "instant fact is not tied to the annual anchor by period end or accession" + ) + if not record.get("filed") or not accession or not period_end: + return "source_context_ambiguous", "filing context is incomplete" + return None, None + + +def _source_refs(records: list[Mapping[str, Any]], source_url: str) -> list[dict[str, Any]]: + refs: list[dict[str, Any]] = [] + for record in records: + refs.append( + { + "source_url": source_url, + "taxonomy": record.get("taxonomy"), + "concept": record.get("concept"), + "unit": record.get("unit"), + "period_start": record.get("period_start"), + "period_end": record.get("period_end"), + "filed": record.get("filed"), + "form": record.get("form"), + "accession": record.get("accession"), + "fiscal_year": record.get("fiscal_year"), + "fiscal_period": record.get("fiscal_period"), + } + ) + return refs + + +def _compare_field( + field: str, + *, + canonical_value: Any, + candidate_value: Any, + provenance: Mapping[str, Any], + anchor_period_start: str | None, + anchor_period_end: str | None, + anchor_accession: str | None, + source_url: str, + registry: Mapping[str, Any], +) -> dict[str, Any]: + canonical_value = _json_value(canonical_value) + candidate_value = _json_value(candidate_value) + value_kind = str(provenance.get("value_kind") or "direct") + records = [ + record + for record in provenance.get("records", []) + if isinstance(record, Mapping) + ] + refs = _source_refs(records, source_url) + + if candidate_value is None: + value_status = "missing" + classification = "missing" + blocker = "No supported SEC fact was selected; the value remains unavailable." + else: + value_status = "unchanged" if _values_equal(canonical_value, candidate_value) else "changed" + context_classification, context_blocker = _field_context( + field, + provenance, + anchor_period_start=anchor_period_start, + anchor_period_end=anchor_period_end, + anchor_accession=anchor_accession, + ) + if context_classification: + classification = context_classification + blocker = context_blocker or "Filing context requires review." + elif value_kind == "derived": + classification = "derived_scope_review_required" + blocker = "Calculated value is not an SEC-reported fact and its exact field scope is not approved." + else: + required_field = DIRECT_RIGHTS_FIELDS.get(field, field) + review = review_commercial_field_scope( + registry, + "sec_companyfacts", + [required_field], + ) + if review.commercial_evidence_ready: + classification = "approved_direct" + blocker = "none" + else: + classification = "unsupported" + blocker = f"Direct SEC field {required_field} is outside the registered commercial field scope." + + first_ref = refs[0] if refs else {} + return { + "field": field, + "canonical_value": canonical_value, + "candidate_value": candidate_value, + "value_status": value_status, + "value_kind": value_kind, + "classification": classification, + "publishability_blocker": blocker, + "period_start": first_ref.get("period_start"), + "period_end": first_ref.get("period_end"), + "filing_date": first_ref.get("filed"), + "accession": first_ref.get("accession"), + "form": first_ref.get("form"), + "source_refs": refs, + } + + +def _compare_source_component( + field: str, + *, + component: Mapping[str, Any], + anchor_period_start: str | None, + anchor_period_end: str | None, + anchor_accession: str | None, + source_url: str, + registry: Mapping[str, Any], + canonical_columns: set[str], +) -> dict[str, Any]: + row = _compare_field( + field, + canonical_value=None, + candidate_value=component.get("value"), + provenance=component, + anchor_period_start=anchor_period_start, + anchor_period_end=anchor_period_end, + anchor_accession=anchor_accession, + source_url=source_url, + registry=registry, + ) + row["value_status"] = "not_canonical" + row["schema_status"] = ( + "existing_canonical_not_produced" + if field in canonical_columns + else "candidate_component_not_canonical" + ) + if row["classification"] == "approved_direct": + row["publishability_blocker"] = ( + "Direct SEC field is approved, but adding a canonical column requires a separate schema decision." + ) + return row + + +def _ticker_failure(ticker: str, status: str, blocker: str) -> dict[str, Any]: + return { + "ticker": ticker, + "status": status, + "blocker": blocker, + "canonical_period_end": None, + "candidate_period_end": None, + "canonical_period_status": "unavailable", + "fields": [], + } + + +def build_sec_fundamentals_preview( + tickers: str | Iterable[str], + *, + canonical_path: str | Path = "data/fundamentals.csv", + staged_path: str | Path = "data/imports/fundamentals.csv", + rights_path: str | Path = DEFAULT_REGISTRY_PATH, + user_agent: str | None = None, + cache_dir: str | Path = "data/cache/sec", + sleep_seconds: float = 0.2, + ticker_map_fetcher: Callable[[str, str, float], Any] | None = None, + companyfacts_fetcher: Callable[[str, str, float], Any] | None = None, +) -> dict[str, Any]: + requested = parse_preview_tickers(tickers) + canonical, canonical_columns = _read_canonical(Path(canonical_path)) + staged_columns = _read_header(Path(staged_path)) + registry = load_source_rights_registry(Path(rights_path)) + ticker_map = load_sec_ticker_map( + cache_dir=cache_dir, + user_agent=user_agent, + sleep_seconds=sleep_seconds, + fetcher=ticker_map_fetcher, + cache=False, + ) + + results: list[dict[str, Any]] = [] + for ticker in requested: + canonical_present = ticker in canonical.index + cik = resolve_ticker_to_cik(ticker, ticker_map) + if cik is None: + results.append( + _ticker_failure( + ticker, + "cik_unresolved", + "No official SEC ticker-to-CIK mapping was found.", + ) + ) + continue + source_url = SEC_COMPANYFACTS_URL.format(cik=cik) + try: + payload = fetch_companyfacts( + cik, + user_agent, + cache=False, + cache_dir=cache_dir, + sleep_seconds=sleep_seconds, + fetcher=companyfacts_fetcher, + ) + except (RuntimeError, TimeoutError, OSError, json.JSONDecodeError) as exc: + results.append(_ticker_failure(ticker, "fetch_failed", str(exc))) + continue + if not _valid_companyfacts_payload(payload): + results.append( + _ticker_failure( + ticker, + "invalid_payload", + "SEC Companyfacts payload was malformed or missing its facts mapping.", + ) + ) + continue + payload_cik = str(payload.get("cik", "")).strip() + normalized_payload_cik = ( + str(int(payload_cik)).zfill(10) if payload_cik.isdigit() else payload_cik + ) + if normalized_payload_cik != str(cik).zfill(10): + results.append( + _ticker_failure( + ticker, + "source_context_ambiguous", + "SEC Companyfacts CIK does not match the official ticker-map CIK.", + ) + ) + continue + + try: + extracted = extract_fundamentals_from_companyfacts(dict(payload)) + except (AttributeError, KeyError, TypeError, ValueError): + results.append( + _ticker_failure( + ticker, + "invalid_payload", + "SEC Companyfacts payload could not be interpreted safely.", + ) + ) + continue + provenance = extracted.get("_field_provenance", {}) + source_components = extracted.get("_source_components", {}) + revenue_records = provenance.get("revenue", {}).get("records", []) + anchor_record = revenue_records[0] if revenue_records else {} + anchor_valid = bool( + extracted.get("revenue") is not None + and isinstance(anchor_record, Mapping) + and _valid_annual_anchor(anchor_record) + ) + anchor_period_end = ( + _json_value(anchor_record.get("period_end")) + if anchor_valid + else None + ) + anchor_period_start = ( + _json_value(anchor_record.get("period_start")) + if anchor_valid + else None + ) + anchor_filing_date = ( + _json_value(anchor_record.get("filed")) if anchor_period_end else None + ) + anchor_accession = ( + _json_value(anchor_record.get("accession")) if anchor_period_end else None + ) + canonical_row = canonical.loc[ticker] if canonical_present else None + if isinstance(canonical_row, pd.DataFrame): + results.append( + _ticker_failure( + ticker, + "canonical_row_ambiguous", + "Canonical fundamentals contains duplicate ticker rows.", + ) + ) + continue + canonical_period_end = ( + _json_value(canonical_row.get("as_of_date")) + if canonical_row is not None + else None + ) + field_rows = [ + _compare_field( + field, + canonical_value=( + canonical_row.get(field) if canonical_row is not None else None + ), + candidate_value=( + extracted.get(field) + if field in extracted + else source_components.get(field, {}).get("value") + ), + provenance=( + provenance.get(field, {}) + if field in provenance + else source_components.get(field, {}) + ), + anchor_period_start=anchor_period_start, + anchor_period_end=anchor_period_end, + anchor_accession=anchor_accession, + source_url=source_url, + registry=registry, + ) + for field in PREVIEW_FIELDS + ] + source_component_rows = [ + _compare_source_component( + field, + component=source_components.get(field, {}), + anchor_period_start=anchor_period_start, + anchor_period_end=anchor_period_end, + anchor_accession=anchor_accession, + source_url=source_url, + registry=registry, + canonical_columns=set(canonical_columns), + ) + for field in CANDIDATE_COMPONENT_FIELDS + ] + future_apply_candidate_fields = [ + row["field"] + for row in field_rows + if row["classification"] == "approved_direct" + and row["value_status"] == "changed" + ] + future_apply_proposal_status = ( + "owner_review_required" + if canonical_present and future_apply_candidate_fields + else "blocked" + ) + results.append( + { + "ticker": ticker, + "status": ( + "compared" if canonical_present else "compared_canonical_missing" + ), + "blocker": ( + "Canonical apply and source-rights decisions remain separately gated." + if canonical_present + else "Canonical fundamentals row is unavailable; a row-level apply proposal is blocked." + ), + "canonical_period_end": canonical_period_end, + "candidate_period_end": anchor_period_end, + "candidate_filing_date": anchor_filing_date, + "candidate_accession": anchor_accession, + "candidate_source_url": source_url, + "canonical_period_status": ( + "aligned" + if canonical_period_end and canonical_period_end == anchor_period_end + else "period_mismatch" + if canonical_period_end + else "unavailable" + ), + "future_apply_candidate_fields": future_apply_candidate_fields, + "future_apply_proposal_status": future_apply_proposal_status, + "source_components": source_component_rows, + "fields": field_rows, + } + ) + + staged_extra = sorted(set(staged_columns) - set(canonical_columns)) + component_extra = sorted( + set(CANDIDATE_COMPONENT_FIELDS) - set(canonical_columns) + ) + canonical_not_produced = sorted( + set(canonical_columns) + - set(PREVIEW_FIELDS) + - { + "ticker", + "source", + "as_of_date", + "sec_cik", + "sec_form", + "sec_filed_date", + "sec_accession", + "sec_fact_warnings", + "sec_entity_name", + } + ) + return { + "status": "inspection_only", + "requested_tickers": requested, + "source": "sec_companyfacts", + "source_rights_mutated": False, + "canonical_apply_authorized": False, + "repository_writes": [], + "schema_delta": { + "staged_extra_columns": staged_extra, + "candidate_component_extra_columns": component_extra, + "canonical_columns_not_produced": canonical_not_produced, + "full_row_rewrite_risk": bool(staged_extra or canonical_not_produced), + }, + "tickers": results, + } + + +def render_sec_fundamentals_preview(result: Mapping[str, Any]) -> str: + return json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Compare official SEC annual fundamentals in memory without writes." + ) + parser.add_argument("--tickers", required=True) + parser.add_argument("--canonical-path", type=Path, default=Path("data/fundamentals.csv")) + parser.add_argument( + "--staged-path", + type=Path, + default=Path("data/imports/fundamentals.csv"), + ) + parser.add_argument("--rights-path", type=Path, default=DEFAULT_REGISTRY_PATH) + parser.add_argument("--sec-user-agent") + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + try: + result = build_sec_fundamentals_preview( + args.tickers, + canonical_path=args.canonical_path, + staged_path=args.staged_path, + rights_path=args.rights_path, + user_agent=args.sec_user_agent, + ) + except ValueError as exc: + parser.error(str(exc)) + print(render_sec_fundamentals_preview(result)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_launchers.py b/tests/test_launchers.py index a2ec2c6d..c3359318 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -368,6 +368,36 @@ def test_full_help_keeps_the_five_primary_readiness_boundaries_separate(): assert boundary in advanced_readiness +def test_sec_fundamentals_preview_is_explicit_capped_and_no_write(): + makefile = Path("Makefile").read_text(encoding="utf-8") + block = _make_target_block(makefile, "sec-fundamentals-preview") + + assert "TICKERS is required" in block + assert ( + 'PYTHONDONTWRITEBYTECODE=1 python3 -m src.sec_fundamentals_preview --tickers "$(TICKERS)"' + in block + ) + assert "--output" not in block + for forbidden in ( + "sec-stage", + "imports-apply", + "readiness-materialize", + "readiness-release-record", + "yfinance", + "yahoo", + "stooq", + "fmp", + "alpha_vantage", + "finnhub", + ): + assert forbidden not in block + + assert ( + "make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" + in makefile + ) + + def test_reviewed_batch_packet_targets_forward_one_named_profile(): makefile = Path("Makefile").read_text(encoding="utf-8") for target in ("reviewed-batch", "fundamentals-batch-proof", "peer-batch-proof"): diff --git a/tests/test_sec_companyfacts.py b/tests/test_sec_companyfacts.py index 459efa32..b429c87e 100644 --- a/tests/test_sec_companyfacts.py +++ b/tests/test_sec_companyfacts.py @@ -301,6 +301,135 @@ def should_not_fetch(*_args, **_kwargs): assert payload["entityName"] == "NVIDIA CORP" +def test_no_cache_sec_requests_do_not_create_cache_paths(tmp_path: Path): + cache_dir = tmp_path / "must-not-exist" + requested_urls: list[str] = [] + + def fake_ticker_fetch(url, *_args): + requested_urls.append(url) + return _sample_ticker_map_payload() + + def fake_companyfacts_fetch(url, *_args): + requested_urls.append(url) + return _sample_companyfacts_payload() + + ticker_map = load_sec_ticker_map( + cache_dir=cache_dir, + user_agent="Test test@example.com", + cache=False, + fetcher=fake_ticker_fetch, + ) + payload = fetch_companyfacts( + "0001045810", + "Test test@example.com", + cache=False, + cache_dir=cache_dir, + fetcher=fake_companyfacts_fetch, + ) + + assert ticker_map["NVDA"]["cik"] == "0001045810" + assert payload["entityName"] == "NVIDIA CORP" + assert requested_urls == [ + "https://www.sec.gov/files/company_tickers.json", + "https://data.sec.gov/api/xbrl/companyfacts/CIK0001045810.json", + ] + assert not cache_dir.exists() + + +def test_extractor_records_direct_and_derived_field_provenance(): + row = extract_fundamentals_from_companyfacts(_sample_companyfacts_payload()) + provenance = row["_field_provenance"] + + assert provenance["revenue"]["value_kind"] == "direct" + assert provenance["revenue"]["records"] == [ + { + "taxonomy": "us-gaap", + "concept": "Revenues", + "unit": "USD", + "period_start": "2025-01-01", + "period_end": "2025-12-31", + "filed": "2026-02-20", + "form": "10-K", + "accession": "0001045810-26-000001", + "fiscal_year": 2025, + "fiscal_period": "FY", + } + ] + assert provenance["free_cash_flow"]["value_kind"] == "derived" + assert {record["concept"] for record in provenance["free_cash_flow"]["records"]} == { + "NetCashProvidedByUsedInOperatingActivities", + "PaymentsToAcquirePropertyPlantAndEquipment", + } + assert provenance["fcf_margin"]["value_kind"] == "derived" + assert provenance["shares_outstanding"]["value_kind"] == "direct" + assert provenance["debt"]["value_kind"] == "derived" + assert len(provenance["debt"]["records"]) == 3 + components = row["_source_components"] + assert components["cash_from_operations"]["value"] == 250 + assert components["capital_expenditures"]["value"] == 50 + assert components["operating_income"]["value"] == 250 + assert components["net_income"]["value"] == 200 + assert all(component["value_kind"] == "direct" for component in components.values()) + + +def test_extractor_keeps_single_reported_total_debt_direct(): + payload = _sample_companyfacts_payload() + gaap = payload["facts"]["us-gaap"] + for concept in ( + "ShortTermBorrowings", + "LongTermDebtCurrent", + "LongTermDebtNoncurrent", + ): + gaap.pop(concept) + gaap["LongTermDebt"] = { + "units": { + "USD": [ + { + "val": 150, + "end": "2025-12-31", + "fy": 2025, + "fp": "FY", + "form": "10-K", + "filed": "2026-02-20", + "accn": "0001045810-26-000001", + } + ] + } + } + + row = extract_fundamentals_from_companyfacts(payload) + + assert row["debt"] == 150 + assert row["_field_provenance"]["debt"]["value_kind"] == "direct" + assert len(row["_field_provenance"]["debt"]["records"]) == 1 + + +def test_extractor_keeps_single_debt_component_derived_and_incomplete(): + payload = _sample_companyfacts_payload() + gaap = payload["facts"]["us-gaap"] + gaap.pop("ShortTermBorrowings") + gaap.pop("LongTermDebtNoncurrent") + + row = extract_fundamentals_from_companyfacts(payload) + + assert row["debt"] == 20 + assert row["_field_provenance"]["debt"]["value_kind"] == "derived" + assert len(row["_field_provenance"]["debt"]["records"]) == 1 + + +def test_staging_rows_omit_private_field_provenance(tmp_path: Path): + result = build_sec_fundamentals_rows( + ["NVDA"], + user_agent="Test test@example.com", + cache_dir=tmp_path / "cache", + ticker_map={"NVDA": {"ticker": "NVDA", "cik": "0001045810"}}, + companyfacts_fetcher=lambda *_: _sample_companyfacts_payload(), + ) + + assert "_field_provenance" not in result["rows"][0] + assert "_source_components" not in result["rows"][0] + + def test_tiny_sec_ticker_map_cache_refreshes(monkeypatch, tmp_path: Path): cache_dir = tmp_path / "cache" ticker_cache = cache_dir / "company_tickers.json" diff --git a/tests/test_sec_fundamentals_preview.py b/tests/test_sec_fundamentals_preview.py new file mode 100644 index 00000000..88564b1c --- /dev/null +++ b/tests/test_sec_fundamentals_preview.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd +import pytest + +from src.sec_fundamentals_preview import ( + build_sec_fundamentals_preview, + parse_preview_tickers, + render_sec_fundamentals_preview, +) + + +def _ticker_map_payload(): + return { + "0": {"cik_str": 320193, "ticker": "AAPL", "title": "APPLE INC"}, + "1": {"cik_str": 1018724, "ticker": "AMZN", "title": "AMAZON COM INC"}, + "2": {"cik_str": 1652044, "ticker": "GOOG", "title": "ALPHABET INC"}, + } + + +def _record( + value, + *, + concept_period="2025-09-27", + start="2024-09-29", + unit="USD", + accession="0000320193-25-000079", +): + return { + "val": value, + "start": start, + "end": concept_period, + "fy": 2025, + "fp": "FY", + "form": "10-K", + "filed": "2025-10-31", + "accn": accession, + "_unit": unit, + } + + +def _fact(record): + unit = record.pop("_unit", "USD") + return {"units": {unit: [record]}} + + +def _companyfacts_payload( + *, + include_shares=True, + eps_period="2025-09-27", + cik=320193, + accession="0000320193-25-000079", +): + revenue_latest = _record(416_161_000_000, accession=accession) + revenue_prior = _record( + 391_035_000_000, + concept_period="2024-09-28", + start="2023-10-01", + accession=accession, + ) + facts = { + "us-gaap": { + "Revenues": {"units": {"USD": [revenue_latest, revenue_prior]}}, + "NetIncomeLoss": _fact(_record(112_010_000_000, accession=accession)), + "EarningsPerShareDiluted": _fact( + _record( + 7.46, + concept_period=eps_period, + unit="USD/shares", + accession=accession, + ) + ), + "NetCashProvidedByUsedInOperatingActivities": _fact( + _record(111_482_000_000, accession=accession) + ), + "PaymentsToAcquirePropertyPlantAndEquipment": _fact( + _record(12_715_000_000, accession=accession) + ), + "OperatingIncomeLoss": _fact( + _record(133_050_000_000, accession=accession) + ), + "CashAndCashEquivalentsAtCarryingValue": _fact( + _record(35_934_000_000, start=None, accession=accession) + ), + "LongTermDebt": _fact( + _record(82_714_000_000, start=None, accession=accession) + ), + }, + "dei": {}, + } + if include_shares: + facts["dei"]["EntityCommonStockSharesOutstanding"] = _fact( + _record( + 14_687_356_000, + start=None, + unit="shares", + accession=accession, + ) + ) + return {"cik": cik, "entityName": "TEST COMPANY", "facts": facts} + + +def _canonical_file(path: Path): + pd.DataFrame( + [ + { + "ticker": "AAPL", + "revenue": 265_595_000_000, + "net_income": 100_000_000_000, + "eps": 7.46, + "free_cash_flow": 98_767_000_000, + "shares_outstanding": 14_687_356_000, + "source": "sec_companyfacts", + "as_of_date": "2018-09-29", + }, + {"ticker": "AMZN", "revenue": 1, "as_of_date": "2024-12-31"}, + {"ticker": "GOOG", "revenue": 1, "as_of_date": "2024-12-31"}, + ] + ).to_csv(path, index=False) + + +def _build(tmp_path: Path, *, payload=None, staged=True): + canonical = tmp_path / "fundamentals.csv" + _canonical_file(canonical) + staged_path = tmp_path / "staged.csv" + if staged: + staged_path.write_text("ticker,revenue,currency\nAAPL,1,USD\n", encoding="utf-8") + requested_urls: list[str] = [] + + def ticker_fetcher(url, *_args): + requested_urls.append(url) + return _ticker_map_payload() + + def facts_fetcher(url, *_args): + requested_urls.append(url) + return payload if payload is not None else _companyfacts_payload() + + result = build_sec_fundamentals_preview( + "AAPL", + canonical_path=canonical, + staged_path=staged_path, + user_agent="Test test@example.com", + ticker_map_fetcher=ticker_fetcher, + companyfacts_fetcher=facts_fetcher, + cache_dir=tmp_path / "must-not-exist", + ) + return result, requested_urls + + +def test_preview_requires_explicit_tickers_and_caps_unique_cohort_at_five(): + with pytest.raises(ValueError, match="explicit"): + parse_preview_tickers("") + assert parse_preview_tickers("aapl, AMZN,aapl,goog") == ["AAPL", "AMZN", "GOOG"] + with pytest.raises(ValueError, match="five"): + parse_preview_tickers("A,B,C,D,E,F") + + +def test_preview_uses_only_official_sec_endpoints_and_writes_no_cache(tmp_path: Path): + result, urls = _build(tmp_path) + + assert result["status"] == "inspection_only" + assert urls == [ + "https://www.sec.gov/files/company_tickers.json", + "https://data.sec.gov/api/xbrl/companyfacts/CIK0000320193.json", + ] + assert all(url.startswith(("https://www.sec.gov/", "https://data.sec.gov/")) for url in urls) + assert not (tmp_path / "must-not-exist").exists() + assert sorted(path.name for path in tmp_path.iterdir()) == [ + "fundamentals.csv", + "staged.csv", + ] + + +def test_preview_preserves_existing_cache_and_input_bytes(tmp_path: Path): + cache_dir = tmp_path / "cache" + ticker_cache = cache_dir / "company_tickers.json" + facts_cache = cache_dir / "companyfacts" / "CIK0000320193.json" + facts_cache.parent.mkdir(parents=True) + ticker_cache.write_bytes(b"user-owned-ticker-cache") + facts_cache.write_bytes(b"user-owned-companyfacts-cache") + canonical = tmp_path / "fundamentals.csv" + staged = tmp_path / "staged.csv" + _canonical_file(canonical) + staged.write_text("ticker,revenue,currency\nAAPL,1,USD\n", encoding="utf-8") + before = { + path: path.read_bytes() for path in (ticker_cache, facts_cache, canonical, staged) + } + + build_sec_fundamentals_preview( + "AAPL", + canonical_path=canonical, + staged_path=staged, + user_agent="Test test@example.com", + ticker_map_fetcher=lambda *_: _ticker_map_payload(), + companyfacts_fetcher=lambda *_: _companyfacts_payload(), + cache_dir=cache_dir, + ) + + assert {path: path.read_bytes() for path in before} == before + + +def test_preview_exposes_aapl_mixed_canonical_period_and_field_classifications(tmp_path: Path): + result, _ = _build(tmp_path) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + + assert ticker["ticker"] == "AAPL" + assert ticker["candidate_period_end"] == "2025-09-27" + assert ticker["canonical_period_end"] == "2018-09-29" + assert ticker["canonical_period_status"] == "period_mismatch" + assert fields["revenue"]["candidate_value"] == 416_161_000_000 + assert fields["revenue"]["value_status"] == "changed" + assert fields["revenue"]["classification"] == "approved_direct" + assert fields["revenue"]["value_kind"] == "direct" + assert fields["revenue"]["accession"] == "0000320193-25-000079" + assert fields["eps"]["value_status"] == "unchanged" + assert fields["eps"]["classification"] == "unsupported" + assert fields["free_cash_flow"]["classification"] == "derived_scope_review_required" + assert fields["free_cash_flow"]["value_kind"] == "derived" + assert fields["fcf_margin"]["classification"] == "derived_scope_review_required" + assert fields["shares_outstanding"]["classification"] == "approved_direct" + assert all(row["publishability_blocker"] for row in ticker["fields"] if row["classification"] != "approved_direct") + components = {row["field"]: row for row in ticker["source_components"]} + assert components["cash_from_operations"]["candidate_value"] == 111_482_000_000 + assert components["cash_from_operations"]["classification"] == "approved_direct" + assert components["capital_expenditures"]["classification"] == "approved_direct" + assert components["operating_income"]["classification"] == "approved_direct" + assert components["net_income"]["classification"] == "unsupported" + assert components["net_income"]["schema_status"] == "existing_canonical_not_produced" + assert all( + components[field]["schema_status"] == "candidate_component_not_canonical" + for field in ( + "cash_from_operations", + "capital_expenditures", + "operating_income", + ) + ) + assert "net_income" not in fields + + +def test_preview_blocks_mixed_candidate_periods(tmp_path: Path): + payload = _companyfacts_payload(eps_period="2024-09-28") + payload["facts"]["us-gaap"]["EarningsPerShareDiluted"]["units"][ + "USD/shares" + ][0]["start"] = "2023-10-01" + result, _ = _build(tmp_path, payload=payload) + fields = {row["field"]: row for row in result["tickers"][0]["fields"]} + + assert fields["eps"]["classification"] == "period_conflict" + assert fields["eps"]["candidate_value"] == 7.46 + assert "annual anchor" in fields["eps"]["publishability_blocker"] + assert "2024-09-29" in fields["eps"]["publishability_blocker"] + + +def test_preview_labels_aggregated_debt_derived_but_single_total_direct( + tmp_path: Path, +): + direct_result, _ = _build(tmp_path) + direct_fields = { + row["field"]: row for row in direct_result["tickers"][0]["fields"] + } + assert direct_fields["debt"]["value_kind"] == "direct" + assert direct_fields["debt"]["classification"] == "unsupported" + + payload = _companyfacts_payload() + gaap = payload["facts"]["us-gaap"] + gaap.pop("LongTermDebt") + gaap["ShortTermBorrowings"] = _fact(_record(10_000_000_000, start=None)) + gaap["LongTermDebtNoncurrent"] = _fact( + _record(72_714_000_000, start=None) + ) + aggregated_result, _ = _build(tmp_path, payload=payload) + aggregated_fields = { + row["field"]: row for row in aggregated_result["tickers"][0]["fields"] + } + + assert aggregated_fields["debt"]["candidate_value"] == 82_714_000_000 + assert aggregated_fields["debt"]["value_kind"] == "derived" + assert ( + aggregated_fields["debt"]["classification"] + == "derived_scope_review_required" + ) + + +def test_preview_rejects_long_duration_quarterly_fact_as_annual_anchor( + tmp_path: Path, +): + payload = _companyfacts_payload() + revenue_records = payload["facts"]["us-gaap"]["Revenues"]["units"]["USD"] + revenue_records[0]["form"] = "10-Q" + revenue_records[0]["start"] = "2024-01-01" + shares = payload["facts"]["dei"]["EntityCommonStockSharesOutstanding"] + shares["units"]["shares"][0]["val"] = 14_000_000_000 + + result, _ = _build(tmp_path, payload=payload) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + + assert fields["revenue"]["classification"] == "period_conflict" + assert fields["shares_outstanding"]["classification"] == "source_context_ambiguous" + assert "annual filing" in fields["revenue"]["publishability_blocker"] + assert ticker["future_apply_candidate_fields"] == [] + assert ticker["future_apply_proposal_status"] == "blocked" + + +def test_preview_rejects_short_duration_fact_even_when_labelled_annual( + tmp_path: Path, +): + payload = _companyfacts_payload() + revenue = payload["facts"]["us-gaap"]["Revenues"]["units"]["USD"][0] + revenue["start"] = "2025-07-01" + shares = payload["facts"]["dei"]["EntityCommonStockSharesOutstanding"] + shares["units"]["shares"][0]["val"] = 14_000_000_000 + + result, _ = _build(tmp_path, payload=payload) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + + assert ticker["candidate_period_end"] is None + assert fields["revenue"]["classification"] == "period_conflict" + assert "annual duration" in fields["revenue"]["publishability_blocker"] + assert fields["shares_outstanding"]["classification"] == "source_context_ambiguous" + assert ticker["future_apply_candidate_fields"] == [] + + +def test_preview_rejects_same_end_but_different_annual_period_start( + tmp_path: Path, +): + payload = _companyfacts_payload() + operating_income = payload["facts"]["us-gaap"]["OperatingIncomeLoss"][ + "units" + ]["USD"][0] + operating_income["start"] = "2024-08-01" + + result, _ = _build(tmp_path, payload=payload) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + components = {row["field"]: row for row in ticker["source_components"]} + + assert fields["operating_margin"]["classification"] == "period_conflict" + assert "period start" in fields["operating_margin"]["publishability_blocker"] + assert components["operating_income"]["classification"] == "period_conflict" + assert "period start" in components["operating_income"]["publishability_blocker"] + + +def test_revenue_growth_requires_adjacent_complete_annual_records(tmp_path: Path): + gap_payload = _companyfacts_payload() + gap_prior = gap_payload["facts"]["us-gaap"]["Revenues"]["units"]["USD"][1] + gap_prior.update( + { + "start": "2022-01-01", + "end": "2022-12-31", + "fy": 2022, + "filed": "2023-02-01", + "accn": "0000320193-23-000001", + } + ) + gap_result, _ = _build(tmp_path, payload=gap_payload) + gap_growth = { + row["field"]: row for row in gap_result["tickers"][0]["fields"] + }["revenue_growth"] + + assert gap_growth["classification"] == "period_conflict" + assert "adjacent" in gap_growth["publishability_blocker"] + + incomplete_payload = _companyfacts_payload() + incomplete_prior = incomplete_payload["facts"]["us-gaap"]["Revenues"]["units"]["USD"][1] + incomplete_prior["accn"] = None + incomplete_result, _ = _build(tmp_path, payload=incomplete_payload) + incomplete_growth = { + row["field"]: row + for row in incomplete_result["tickers"][0]["fields"] + }["revenue_growth"] + + assert incomplete_growth["classification"] == "source_context_ambiguous" + assert "complete" in incomplete_growth["publishability_blocker"] + + +def test_preview_does_not_substitute_another_fact_for_missing_revenue_anchor( + tmp_path: Path, +): + payload = _companyfacts_payload() + payload["facts"]["us-gaap"].pop("Revenues") + + result, _ = _build(tmp_path, payload=payload) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + components = {row["field"]: row for row in ticker["source_components"]} + + assert ticker["candidate_period_end"] is None + assert ticker["candidate_accession"] is None + assert fields["revenue"]["classification"] == "missing" + assert components["net_income"]["classification"] == "period_conflict" + assert ticker["future_apply_candidate_fields"] == [] + + +def test_preview_keeps_missing_goog_shares_unavailable(tmp_path: Path): + canonical = tmp_path / "fundamentals.csv" + _canonical_file(canonical) + result = build_sec_fundamentals_preview( + "GOOG", + canonical_path=canonical, + staged_path=tmp_path / "missing-staged.csv", + user_agent="Test test@example.com", + ticker_map_fetcher=lambda *_: _ticker_map_payload(), + companyfacts_fetcher=lambda *_: _companyfacts_payload( + include_shares=False, + cik=1652044, + accession="0001652044-25-000100", + ), + cache_dir=tmp_path / "must-not-exist", + ) + fields = {row["field"]: row for row in result["tickers"][0]["fields"]} + + assert fields["shares_outstanding"]["candidate_value"] is None + assert fields["shares_outstanding"]["classification"] == "missing" + assert fields["shares_outstanding"]["value_status"] == "missing" + + +def test_preview_inspects_sec_candidate_but_blocks_apply_when_canonical_row_is_missing( + tmp_path: Path, +): + canonical = tmp_path / "fundamentals.csv" + pd.DataFrame( + [{"ticker": "AAPL", "revenue": 1, "as_of_date": "2024-01-01"}] + ).to_csv(canonical, index=False) + result = build_sec_fundamentals_preview( + "GOOG", + canonical_path=canonical, + staged_path=tmp_path / "missing.csv", + user_agent="Test test@example.com", + ticker_map_fetcher=lambda *_: _ticker_map_payload(), + companyfacts_fetcher=lambda *_: _companyfacts_payload( + include_shares=False, + cik=1652044, + accession="0001652044-25-000100", + ), + cache_dir=tmp_path / "must-not-exist", + ) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + + assert ticker["status"] == "compared_canonical_missing" + assert ticker["canonical_period_status"] == "unavailable" + assert ticker["future_apply_proposal_status"] == "blocked" + assert fields["revenue"]["candidate_value"] == 416_161_000_000 + assert fields["revenue"]["canonical_value"] is None + assert fields["shares_outstanding"]["classification"] == "missing" + + +def test_preview_rejects_companyfacts_from_a_different_cik(tmp_path: Path): + result, _ = _build( + tmp_path, + payload=_companyfacts_payload(cik=1652044), + ) + + assert result["tickers"][0]["status"] == "source_context_ambiguous" + assert result["tickers"][0]["fields"] == [] + assert "CIK" in result["tickers"][0]["blocker"] + + +def test_preview_fails_closed_for_malformed_companyfacts(tmp_path: Path): + result, _ = _build(tmp_path, payload={"facts": []}) + + assert result["tickers"][0]["status"] == "invalid_payload" + assert result["tickers"][0]["fields"] == [] + assert "malformed" in result["tickers"][0]["blocker"].lower() + + +def test_malformed_first_ticker_does_not_abort_later_valid_ticker(tmp_path: Path): + canonical = tmp_path / "fundamentals.csv" + _canonical_file(canonical) + + def facts_fetcher(url, *_args): + if "CIK0000320193" in url: + return { + "cik": 320193, + "facts": { + "us-gaap": { + "Revenues": {"units": {"USD": "malformed"}} + } + }, + } + return _companyfacts_payload( + cik=1018724, + accession="0001018724-25-000100", + ) + + result = build_sec_fundamentals_preview( + "AAPL,AMZN", + canonical_path=canonical, + staged_path=tmp_path / "missing.csv", + user_agent="Test test@example.com", + ticker_map_fetcher=lambda *_: _ticker_map_payload(), + companyfacts_fetcher=facts_fetcher, + cache_dir=tmp_path / "must-not-exist", + ) + + assert [ticker["ticker"] for ticker in result["tickers"]] == ["AAPL", "AMZN"] + assert result["tickers"][0]["status"] == "invalid_payload" + assert result["tickers"][0]["fields"] == [] + assert result["tickers"][1]["status"] == "compared" + assert result["tickers"][1]["fields"] + + +def test_preview_reports_staged_schema_expansion_and_is_deterministic(tmp_path: Path): + result, _ = _build(tmp_path) + + assert result["schema_delta"]["staged_extra_columns"] == ["currency"] + assert "cash_from_operations" in result["schema_delta"]["candidate_component_extra_columns"] + assert "net_income" not in result["schema_delta"]["candidate_component_extra_columns"] + assert "net_income" in result["schema_delta"]["canonical_columns_not_produced"] + rendered = render_sec_fundamentals_preview(result) + assert json.loads(rendered) == result + assert rendered == render_sec_fundamentals_preview(result) From a4663b08b309a3dc38b4ca17d4d8ae3f5b3646b3 Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Sat, 15 Aug 2026 21:23:07 -0400 Subject: [PATCH 4/5] Document no-write SEC preview --- README.md | 2 +- tests/test_public_v1_release_docs.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 769eb91a..ca10126f 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The report is not a black box: local data rows provide inputs, and project rules ## Current Snapshot The local sample tracks a broad stock universe, with a smaller subset ready for each analysis feature. Exact universe and ready counts can change after local refresh/import work, so use `make readiness-ops-center` for current lane truth. Treat `make status-check TOP_N=5` and dashboard counts as saved generated-snapshot context, not current-market freshness proof. Read the counts in three layers: master universe for broad coverage planning, active universe for the demo/research workflow, and analysis-ready subsets for DCF, peer context, or candidate review. A tracked ticker is not automatically ready for every analysis family; blocked rows stay visibly locked. -Visitor status: the product workflow, dashboard, single-stock reports, readiness gates, visitor path, and public checks are working. Broad fundamentals, DCF, peers, earnings, and analyst estimates remain visibly blocked by missing trusted data until trusted rows exist, so those gaps should be read as source-proof work rather than broken analysis. +Visitor status: the product workflow, dashboard, single-stock reports, readiness gates, visitor path, and public checks are working. Broad fundamentals, DCF, peers, earnings, and analyst estimates remain visibly blocked by missing trusted data until trusted rows exist, so those gaps should be read as source-proof work rather than broken analysis. **No-key SEC actuals inspection:** run `make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG` to compare annual facts from official SEC endpoints for at most five explicit tickers. The command is inspection-only: it writes no cache, import, canonical, readiness, or output files and does not authorize a data apply. Only direct fields already allowed by the registered source scope can become future owner-review candidates; Derived and out-of-scope fields remain blocked, and the result does not activate product readiness. ## External Reviewer Handoff Use this as the short GitHub/LinkedIn review path before reading operator detail: | Question | Short answer | diff --git a/tests/test_public_v1_release_docs.py b/tests/test_public_v1_release_docs.py index 941625fe..d7e93471 100644 --- a/tests/test_public_v1_release_docs.py +++ b/tests/test_public_v1_release_docs.py @@ -337,6 +337,18 @@ def test_readme_has_compact_current_next_stages_for_external_reviewers(): assert readme.index("## Now / Next / Not Yet") < readme.index("## What You Can Analyze") +def test_readme_explains_no_key_sec_preview_without_claiming_activation(): + readme = _read("README.md") + + assert "**No-key SEC actuals inspection:**" in readme + assert "`make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG`" in readme + assert "official SEC endpoints" in readme + assert "at most five explicit tickers" in readme + assert "writes no cache, import, canonical, readiness, or output files" in readme + assert "does not authorize a data apply" in readme + assert "Derived and out-of-scope fields remain blocked" in readme + + def test_public_status_language_keeps_share_review_ready_local_only(): readme = _read("README.md") From 39d92db1d0d4b7af0ddfd78bc3d99280c4168996 Mon Sep 17 00:00:00 2001 From: davidjiang8888 Date: Tue, 18 Aug 2026 19:23:25 -0400 Subject: [PATCH 5/5] Complete SEC permitted-evidence preview --- Makefile | 4 +- README.md | 2 +- src/sec_fundamentals_preview.py | 281 +++++++++++++++++++++++-- tests/test_launchers.py | 2 +- tests/test_public_v1_release_docs.py | 5 +- tests/test_sec_fundamentals_preview.py | 152 ++++++++++++- 6 files changed, 420 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index f02f9b30..92737790 100644 --- a/Makefile +++ b/Makefile @@ -356,7 +356,7 @@ help-full: @echo "Preview-first fundamentals and universe imports:" @echo " export SEC_USER_AGENT='Name email@example.com'" @echo " make sec-stage TICKERS=NVDA,MSFT" - @echo " make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" + @echo " make sec-fundamentals-preview TICKERS=AAPL,NVDA,AMD Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" @echo " make yfinance-stage TICKERS=NVDA" @echo " make fundamentals-source-ladder TICKERS=NVDA" @echo " Try SEC, yfinance, FMP, Alpha Vantage, then Finnhub before stopping at reviewed blocker evidence" @@ -1275,7 +1275,7 @@ endif .PHONY: sec-fundamentals-preview sec-fundamentals-preview: ifndef TICKERS - $(error TICKERS is required, for example: make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG) + $(error TICKERS is required, for example: make sec-fundamentals-preview TICKERS=AAPL,NVDA,AMD) endif @PYTHONDONTWRITEBYTECODE=1 python3 -m src.sec_fundamentals_preview --tickers "$(TICKERS)" diff --git a/README.md b/README.md index ca10126f..5fab4755 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The report is not a black box: local data rows provide inputs, and project rules ## Current Snapshot The local sample tracks a broad stock universe, with a smaller subset ready for each analysis feature. Exact universe and ready counts can change after local refresh/import work, so use `make readiness-ops-center` for current lane truth. Treat `make status-check TOP_N=5` and dashboard counts as saved generated-snapshot context, not current-market freshness proof. Read the counts in three layers: master universe for broad coverage planning, active universe for the demo/research workflow, and analysis-ready subsets for DCF, peer context, or candidate review. A tracked ticker is not automatically ready for every analysis family; blocked rows stay visibly locked. -Visitor status: the product workflow, dashboard, single-stock reports, readiness gates, visitor path, and public checks are working. Broad fundamentals, DCF, peers, earnings, and analyst estimates remain visibly blocked by missing trusted data until trusted rows exist, so those gaps should be read as source-proof work rather than broken analysis. **No-key SEC actuals inspection:** run `make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG` to compare annual facts from official SEC endpoints for at most five explicit tickers. The command is inspection-only: it writes no cache, import, canonical, readiness, or output files and does not authorize a data apply. Only direct fields already allowed by the registered source scope can become future owner-review candidates; Derived and out-of-scope fields remain blocked, and the result does not activate product readiness. +Visitor status: the product workflow, dashboard, single-stock reports, readiness gates, visitor path, and public checks are working. Broad fundamentals, DCF, peers, earnings, and analyst estimates remain visibly blocked by missing trusted data until trusted rows exist, so those gaps should be read as source-proof work rather than broken analysis. **No-key SEC actuals inspection:** run `make sec-fundamentals-preview TICKERS=AAPL,NVDA,AMD` to compare annual facts from official SEC endpoints for at most five explicit tickers. The command is inspection-only: it writes no cache, import, canonical, readiness, or output files and does not authorize a data apply. The packet records field-level period, unit, retrieval, source-rights, schema, delta, and owner-action evidence. Only direct fields already allowed by the registered source scope can become future owner-review candidates; derived, mixed-unit, out-of-scope, and incoherent-period fields remain blocked, and the result does not activate product readiness. ## External Reviewer Handoff Use this as the short GitHub/LinkedIn review path before reading operator detail: | Question | Short answer | diff --git a/src/sec_fundamentals_preview.py b/src/sec_fundamentals_preview.py index c94d2084..a3c15297 100644 --- a/src/sec_fundamentals_preview.py +++ b/src/sec_fundamentals_preview.py @@ -6,6 +6,7 @@ import json import math import re +from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable, Iterable, Mapping @@ -43,6 +44,7 @@ DIRECT_RIGHTS_FIELDS = { "revenue": "revenue", "shares_outstanding": "shares_outstanding", + "filing_dates": "filing_dates", } CANDIDATE_COMPONENT_FIELDS = ( "net_income", @@ -50,7 +52,21 @@ "capital_expenditures", "operating_income", ) +REGISTERED_ACTIVATION_FIELDS = ( + "revenue", + "shares_outstanding", + "filing_dates", + "operating_income", + "cash_from_operations", + "capital_expenditures", +) _TICKER_PATTERN = re.compile(r"^[A-Z0-9][A-Z0-9.-]*$") +_RATIO_FIELDS = { + "revenue_growth", + "fcf_margin", + "profit_margin", + "operating_margin", +} def parse_preview_tickers(value: str | Iterable[str]) -> list[str]: @@ -96,6 +112,40 @@ def _values_equal(left: Any, right: Any) -> bool: return left == right +def _retrieval_timestamp(value: str | None) -> str: + if value is None: + timestamp = datetime.now(timezone.utc) + else: + try: + timestamp = datetime.fromisoformat( + str(value).strip().replace("Z", "+00:00") + ) + except ValueError as exc: + raise ValueError("retrieval_timestamp must be a timezone-aware ISO timestamp") from exc + if timestamp.tzinfo is None: + raise ValueError("retrieval_timestamp must be a timezone-aware ISO timestamp") + timestamp = timestamp.astimezone(timezone.utc) + return timestamp.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _proposed_delta(canonical_value: Any, candidate_value: Any) -> dict[str, Any]: + canonical_value = _json_value(canonical_value) + candidate_value = _json_value(candidate_value) + numeric_change = None + if ( + isinstance(canonical_value, (int, float)) + and not isinstance(canonical_value, bool) + and isinstance(candidate_value, (int, float)) + and not isinstance(candidate_value, bool) + ): + numeric_change = _json_value(candidate_value - canonical_value) + return { + "from": canonical_value, + "to": candidate_value, + "numeric_change": numeric_change, + } + + def _read_canonical(path: Path) -> tuple[pd.DataFrame, list[str]]: if not path.is_file(): raise ValueError(f"canonical fundamentals file is unavailable: {path}") @@ -250,15 +300,21 @@ def _field_context( return None, None -def _source_refs(records: list[Mapping[str, Any]], source_url: str) -> list[dict[str, Any]]: +def _source_refs( + records: list[Mapping[str, Any]], + source_url: str, + retrieval_timestamp: str, +) -> list[dict[str, Any]]: refs: list[dict[str, Any]] = [] for record in records: refs.append( { "source_url": source_url, + "retrieval_timestamp": retrieval_timestamp, "taxonomy": record.get("taxonomy"), "concept": record.get("concept"), "unit": record.get("unit"), + "underlying_fact_unit": record.get("underlying_fact_unit"), "period_start": record.get("period_start"), "period_end": record.get("period_end"), "filed": record.get("filed"), @@ -271,16 +327,82 @@ def _source_refs(records: list[Mapping[str, Any]], source_url: str) -> list[dict return refs +def _source_units(records: list[Mapping[str, Any]]) -> list[str]: + return sorted( + { + str(record.get("unit") or "").strip() + for record in records + if str(record.get("unit") or "").strip() + } + ) + + +def _field_unit(field: str, source_units: list[str]) -> str | None: + if len(source_units) > 1: + return "mixed" + if field in _RATIO_FIELDS: + return "ratio" + if field == "filing_dates": + return "date" + return source_units[0] if source_units else None + + +def _unit_context( + records: list[Mapping[str, Any]], +) -> tuple[str | None, str | None]: + if not records: + return None, None + if any(not str(record.get("unit") or "").strip() for record in records): + return ( + "source_context_ambiguous", + "Every selected SEC fact requires an explicit source unit.", + ) + if len(_source_units(records)) > 1: + return ( + "unit_conflict", + "Selected SEC facts do not have consistent source units or currency.", + ) + return None, None + + +def _owner_action( + classification: str, + *, + value_status: str, + schema_status: str, +) -> str: + if classification == "approved_direct": + if schema_status != "existing_canonical": + return "approve_canonical_schema_before_any_apply" + if value_status == "changed": + return "review_field_delta_before_separate_apply_authorization" + return "retain_reviewed_source_evidence_no_value_apply_needed" + if classification == "derived_scope_review_required": + return "approve_exact_derived_field_scope_or_keep_withheld" + if classification == "unsupported": + return "approve_exact_registered_field_scope_or_keep_withheld" + if classification == "missing": + return "supply_supported_sec_fact_or_keep_withheld" + if classification == "unit_conflict": + return "supply_unit_coherent_sec_facts_or_keep_withheld" + return "resolve_period_or_source_context_before_any_apply_review" + + def _compare_field( field: str, *, + ticker: str, + cik: str, canonical_value: Any, candidate_value: Any, + canonical_column: str, + schema_status: str, provenance: Mapping[str, Any], anchor_period_start: str | None, anchor_period_end: str | None, anchor_accession: str | None, source_url: str, + retrieval_timestamp: str, registry: Mapping[str, Any], ) -> dict[str, Any]: canonical_value = _json_value(canonical_value) @@ -291,21 +413,29 @@ def _compare_field( for record in provenance.get("records", []) if isinstance(record, Mapping) ] - refs = _source_refs(records, source_url) - + refs = _source_refs(records, source_url, retrieval_timestamp) + source_units = _source_units(records) + required_field = DIRECT_RIGHTS_FIELDS.get(field, field) + rights_review = review_commercial_field_scope( + registry, + "sec_companyfacts", + [required_field], + ) if candidate_value is None: value_status = "missing" classification = "missing" blocker = "No supported SEC fact was selected; the value remains unavailable." else: value_status = "unchanged" if _values_equal(canonical_value, candidate_value) else "changed" - context_classification, context_blocker = _field_context( - field, - provenance, - anchor_period_start=anchor_period_start, - anchor_period_end=anchor_period_end, - anchor_accession=anchor_accession, - ) + context_classification, context_blocker = _unit_context(records) + if context_classification is None: + context_classification, context_blocker = _field_context( + field, + provenance, + anchor_period_start=anchor_period_start, + anchor_period_end=anchor_period_end, + anchor_accession=anchor_accession, + ) if context_classification: classification = context_classification blocker = context_blocker or "Filing context requires review." @@ -313,13 +443,7 @@ def _compare_field( classification = "derived_scope_review_required" blocker = "Calculated value is not an SEC-reported fact and its exact field scope is not approved." else: - required_field = DIRECT_RIGHTS_FIELDS.get(field, field) - review = review_commercial_field_scope( - registry, - "sec_companyfacts", - [required_field], - ) - if review.commercial_evidence_ready: + if rights_review.commercial_evidence_ready: classification = "approved_direct" blocker = "none" else: @@ -328,42 +452,80 @@ def _compare_field( first_ref = refs[0] if refs else {} return { + "ticker": ticker, + "sec_cik": str(cik).zfill(10), "field": field, + "canonical_column": canonical_column, "canonical_value": canonical_value, "candidate_value": candidate_value, + "unit": _field_unit(field, source_units), + "source_units": source_units, "value_status": value_status, "value_kind": value_kind, "classification": classification, + "source_rights_status": rights_review.rights_status, + "commercial_rights_approved": rights_review.commercial_rights_approved, + "required_registered_field": required_field, + "field_scope_status": ( + "approved" if rights_review.commercial_evidence_ready else "review_required" + ), + "missing_registered_fields": list(rights_review.missing_supported_fields), "publishability_blocker": blocker, + "required_owner_action": _owner_action( + classification, + value_status=value_status, + schema_status=schema_status, + ), + "proposed_delta": _proposed_delta(canonical_value, candidate_value), + "retrieval_timestamp": retrieval_timestamp, + "source_url": source_url, "period_start": first_ref.get("period_start"), "period_end": first_ref.get("period_end"), "filing_date": first_ref.get("filed"), "accession": first_ref.get("accession"), "form": first_ref.get("form"), "source_refs": refs, + "schema_status": schema_status, + **( + {"basis_status": "reported_value_no_split_adjustment"} + if field == "shares_outstanding" + else {} + ), } def _compare_source_component( field: str, *, + ticker: str, + cik: str, component: Mapping[str, Any], anchor_period_start: str | None, anchor_period_end: str | None, anchor_accession: str | None, source_url: str, + retrieval_timestamp: str, registry: Mapping[str, Any], canonical_columns: set[str], ) -> dict[str, Any]: row = _compare_field( field, + ticker=ticker, + cik=cik, canonical_value=None, candidate_value=component.get("value"), + canonical_column=field, + schema_status=( + "existing_canonical" + if field in canonical_columns + else "canonical_column_missing" + ), provenance=component, anchor_period_start=anchor_period_start, anchor_period_end=anchor_period_end, anchor_accession=anchor_accession, source_url=source_url, + retrieval_timestamp=retrieval_timestamp, registry=registry, ) row["value_status"] = "not_canonical" @@ -376,6 +538,11 @@ def _compare_source_component( row["publishability_blocker"] = ( "Direct SEC field is approved, but adding a canonical column requires a separate schema decision." ) + row["required_owner_action"] = _owner_action( + row["classification"], + value_status=row["value_status"], + schema_status=row["schema_status"], + ) return row @@ -394,6 +561,7 @@ def _ticker_failure(ticker: str, status: str, blocker: str) -> dict[str, Any]: def build_sec_fundamentals_preview( tickers: str | Iterable[str], *, + retrieval_timestamp: str | None = None, canonical_path: str | Path = "data/fundamentals.csv", staged_path: str | Path = "data/imports/fundamentals.csv", rights_path: str | Path = DEFAULT_REGISTRY_PATH, @@ -404,6 +572,7 @@ def build_sec_fundamentals_preview( companyfacts_fetcher: Callable[[str, str, float], Any] | None = None, ) -> dict[str, Any]: requested = parse_preview_tickers(tickers) + normalized_retrieval_timestamp = _retrieval_timestamp(retrieval_timestamp) canonical, canonical_columns = _read_canonical(Path(canonical_path)) staged_columns = _read_header(Path(staged_path)) registry = load_source_rights_registry(Path(rights_path)) @@ -518,6 +687,8 @@ def build_sec_fundamentals_preview( field_rows = [ _compare_field( field, + ticker=ticker, + cik=cik, canonical_value=( canonical_row.get(field) if canonical_row is not None else None ), @@ -526,6 +697,12 @@ def build_sec_fundamentals_preview( if field in extracted else source_components.get(field, {}).get("value") ), + canonical_column=field, + schema_status=( + "existing_canonical" + if field in canonical_columns + else "canonical_column_missing" + ), provenance=( provenance.get(field, {}) if field in provenance @@ -535,18 +712,55 @@ def build_sec_fundamentals_preview( anchor_period_end=anchor_period_end, anchor_accession=anchor_accession, source_url=source_url, + retrieval_timestamp=normalized_retrieval_timestamp, registry=registry, ) for field in PREVIEW_FIELDS ] + filing_record = dict(anchor_record) if anchor_valid else {} + if filing_record: + filing_record["underlying_fact_unit"] = filing_record.get("unit") + filing_record["unit"] = "date" + field_rows.append( + _compare_field( + "filing_dates", + ticker=ticker, + cik=cik, + canonical_value=( + canonical_row.get("sec_filed_date") + if canonical_row is not None + else None + ), + candidate_value=anchor_filing_date, + canonical_column="sec_filed_date", + schema_status=( + "existing_canonical" + if "sec_filed_date" in canonical_columns + else "canonical_column_missing" + ), + provenance={ + "value_kind": "direct", + "records": [filing_record] if filing_record else [], + }, + anchor_period_start=anchor_period_start, + anchor_period_end=anchor_period_end, + anchor_accession=anchor_accession, + source_url=source_url, + retrieval_timestamp=normalized_retrieval_timestamp, + registry=registry, + ) + ) source_component_rows = [ _compare_source_component( field, + ticker=ticker, + cik=cik, component=source_components.get(field, {}), anchor_period_start=anchor_period_start, anchor_period_end=anchor_period_end, anchor_accession=anchor_accession, source_url=source_url, + retrieval_timestamp=normalized_retrieval_timestamp, registry=registry, canonical_columns=set(canonical_columns), ) @@ -557,6 +771,32 @@ def build_sec_fundamentals_preview( for row in field_rows if row["classification"] == "approved_direct" and row["value_status"] == "changed" + and row["schema_status"] == "existing_canonical" + ] + changed_fields = [ + row["field"] for row in field_rows if row["value_status"] == "changed" + ] + schema_review_required_fields = [ + row["field"] + for row in source_component_rows + if row["classification"] == "approved_direct" + and row["schema_status"] == "candidate_component_not_canonical" + ] + intentionally_withheld_fields = [ + row["field"] + for row in [*field_rows, *source_component_rows] + if row["classification"] != "approved_direct" + ] + registered_field_reviews = [ + { + "field": row["field"], + "classification": row["classification"], + "value_status": row["value_status"], + "schema_status": row["schema_status"], + "required_owner_action": row["required_owner_action"], + } + for row in [*field_rows, *source_component_rows] + if row["field"] in REGISTERED_ACTIVATION_FIELDS ] future_apply_proposal_status = ( "owner_review_required" @@ -566,6 +806,8 @@ def build_sec_fundamentals_preview( results.append( { "ticker": ticker, + "sec_cik": str(cik).zfill(10), + "retrieval_timestamp": normalized_retrieval_timestamp, "status": ( "compared" if canonical_present else "compared_canonical_missing" ), @@ -588,6 +830,10 @@ def build_sec_fundamentals_preview( ), "future_apply_candidate_fields": future_apply_candidate_fields, "future_apply_proposal_status": future_apply_proposal_status, + "changed_fields": changed_fields, + "schema_review_required_fields": schema_review_required_fields, + "intentionally_withheld_fields": intentionally_withheld_fields, + "registered_field_reviews": registered_field_reviews, "source_components": source_component_rows, "fields": field_rows, } @@ -615,6 +861,7 @@ def build_sec_fundamentals_preview( return { "status": "inspection_only", "requested_tickers": requested, + "retrieval_timestamp": normalized_retrieval_timestamp, "source": "sec_companyfacts", "source_rights_mutated": False, "canonical_apply_authorized": False, diff --git a/tests/test_launchers.py b/tests/test_launchers.py index c3359318..34837267 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -393,7 +393,7 @@ def test_sec_fundamentals_preview_is_explicit_capped_and_no_write(): assert forbidden not in block assert ( - "make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" + "make sec-fundamentals-preview TICKERS=AAPL,NVDA,AMD Official SEC annual comparison; max five explicit tickers; no cache, staging, or apply writes" in makefile ) diff --git a/tests/test_public_v1_release_docs.py b/tests/test_public_v1_release_docs.py index d7e93471..fe7d488a 100644 --- a/tests/test_public_v1_release_docs.py +++ b/tests/test_public_v1_release_docs.py @@ -341,12 +341,13 @@ def test_readme_explains_no_key_sec_preview_without_claiming_activation(): readme = _read("README.md") assert "**No-key SEC actuals inspection:**" in readme - assert "`make sec-fundamentals-preview TICKERS=AAPL,AMZN,GOOG`" in readme + assert "`make sec-fundamentals-preview TICKERS=AAPL,NVDA,AMD`" in readme assert "official SEC endpoints" in readme assert "at most five explicit tickers" in readme assert "writes no cache, import, canonical, readiness, or output files" in readme assert "does not authorize a data apply" in readme - assert "Derived and out-of-scope fields remain blocked" in readme + assert "field-level period, unit, retrieval, source-rights, schema, delta, and owner-action evidence" in readme + assert "derived, mixed-unit, out-of-scope, and incoherent-period fields remain blocked" in readme def test_public_status_language_keeps_share_review_ready_local_only(): diff --git a/tests/test_sec_fundamentals_preview.py b/tests/test_sec_fundamentals_preview.py index 88564b1c..391055ca 100644 --- a/tests/test_sec_fundamentals_preview.py +++ b/tests/test_sec_fundamentals_preview.py @@ -115,6 +115,7 @@ def _canonical_file(path: Path): "shares_outstanding": 14_687_356_000, "source": "sec_companyfacts", "as_of_date": "2018-09-29", + "sec_filed_date": "2018-11-05", }, {"ticker": "AMZN", "revenue": 1, "as_of_date": "2024-12-31"}, {"ticker": "GOOG", "revenue": 1, "as_of_date": "2024-12-31"}, @@ -122,7 +123,13 @@ def _canonical_file(path: Path): ).to_csv(path, index=False) -def _build(tmp_path: Path, *, payload=None, staged=True): +def _build( + tmp_path: Path, + *, + payload=None, + staged=True, + retrieval_timestamp=None, +): canonical = tmp_path / "fundamentals.csv" _canonical_file(canonical) staged_path = tmp_path / "staged.csv" @@ -138,6 +145,9 @@ def facts_fetcher(url, *_args): requested_urls.append(url) return payload if payload is not None else _companyfacts_payload() + kwargs = {} + if retrieval_timestamp is not None: + kwargs["retrieval_timestamp"] = retrieval_timestamp result = build_sec_fundamentals_preview( "AAPL", canonical_path=canonical, @@ -146,6 +156,7 @@ def facts_fetcher(url, *_args): ticker_map_fetcher=ticker_fetcher, companyfacts_fetcher=facts_fetcher, cache_dir=tmp_path / "must-not-exist", + **kwargs, ) return result, requested_urls @@ -241,6 +252,139 @@ def test_preview_exposes_aapl_mixed_canonical_period_and_field_classifications(t assert "net_income" not in fields +def test_preview_emits_complete_field_level_review_metadata(tmp_path: Path): + result, _ = _build( + tmp_path, + retrieval_timestamp="2026-08-18T22:15:30Z", + ) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + components = {row["field"]: row for row in ticker["source_components"]} + + assert result["retrieval_timestamp"] == "2026-08-18T22:15:30Z" + assert ticker["sec_cik"] == "0000320193" + assert ticker["changed_fields"] == [ + "revenue", + "revenue_growth", + "fcf_margin", + "profit_margin", + "operating_margin", + "cash", + "debt", + "filing_dates", + ] + assert ticker["schema_review_required_fields"] == [ + "cash_from_operations", + "capital_expenditures", + "operating_income", + ] + assert "eps" in ticker["intentionally_withheld_fields"] + assert "free_cash_flow" in ticker["intentionally_withheld_fields"] + + revenue = fields["revenue"] + assert revenue == { + **revenue, + "ticker": "AAPL", + "sec_cik": "0000320193", + "canonical_column": "revenue", + "unit": "USD", + "source_units": ["USD"], + "retrieval_timestamp": "2026-08-18T22:15:30Z", + "source_rights_status": "approved", + "commercial_rights_approved": True, + "required_registered_field": "revenue", + "field_scope_status": "approved", + "missing_registered_fields": [], + "proposed_delta": { + "from": 265_595_000_000, + "to": 416_161_000_000, + "numeric_change": 150_566_000_000, + }, + "required_owner_action": "review_field_delta_before_separate_apply_authorization", + } + assert revenue["source_refs"][0]["retrieval_timestamp"] == "2026-08-18T22:15:30Z" + + filing_date = fields["filing_dates"] + assert filing_date["canonical_column"] == "sec_filed_date" + assert filing_date["candidate_value"] == "2025-10-31" + assert filing_date["unit"] == "date" + assert filing_date["value_kind"] == "direct" + assert filing_date["classification"] == "approved_direct" + assert filing_date["field_scope_status"] == "approved" + assert filing_date["source_units"] == ["date"] + assert filing_date["source_refs"][0]["taxonomy"] == "us-gaap" + assert filing_date["source_refs"][0]["concept"] == "Revenues" + assert filing_date["source_refs"][0]["unit"] == "date" + assert filing_date["source_refs"][0]["underlying_fact_unit"] == "USD" + assert filing_date["proposed_delta"] == { + "from": "2018-11-05", + "to": "2025-10-31", + "numeric_change": None, + } + assert "filing_dates" in ticker["future_apply_candidate_fields"] + + operating_income = components["operating_income"] + assert operating_income["unit"] == "USD" + assert operating_income["source_rights_status"] == "approved" + assert operating_income["field_scope_status"] == "approved" + assert operating_income["required_owner_action"] == ( + "approve_canonical_schema_before_any_apply" + ) + + shares = fields["shares_outstanding"] + assert shares["basis_status"] == "reported_value_no_split_adjustment" + assert shares["classification"] == "approved_direct" + + +def test_preview_blocks_cross_currency_derived_values(tmp_path: Path): + payload = _companyfacts_payload() + capex = payload["facts"]["us-gaap"][ + "PaymentsToAcquirePropertyPlantAndEquipment" + ]["units"].pop("USD")[0] + payload["facts"]["us-gaap"][ + "PaymentsToAcquirePropertyPlantAndEquipment" + ]["units"]["EUR"] = [capex] + + result, _ = _build(tmp_path, payload=payload) + fields = {row["field"]: row for row in result["tickers"][0]["fields"]} + + assert fields["free_cash_flow"]["source_units"] == ["EUR", "USD"] + assert fields["free_cash_flow"]["unit"] == "mixed" + assert fields["free_cash_flow"]["classification"] == "unit_conflict" + assert "consistent source units" in fields["free_cash_flow"][ + "publishability_blocker" + ] + assert fields["fcf_margin"]["classification"] == "unit_conflict" + + +def test_preview_does_not_make_missing_canonical_columns_apply_candidates( + tmp_path: Path, +): + canonical = tmp_path / "fundamentals.csv" + pd.DataFrame( + [{"ticker": "AAPL", "revenue": 1, "as_of_date": "2024-01-01"}] + ).to_csv(canonical, index=False) + result = build_sec_fundamentals_preview( + "AAPL", + retrieval_timestamp="2026-08-18T22:15:30Z", + canonical_path=canonical, + staged_path=tmp_path / "missing.csv", + user_agent="Test test@example.com", + ticker_map_fetcher=lambda *_: _ticker_map_payload(), + companyfacts_fetcher=lambda *_: _companyfacts_payload(), + cache_dir=tmp_path / "must-not-exist", + ) + ticker = result["tickers"][0] + fields = {row["field"]: row for row in ticker["fields"]} + + assert fields["filing_dates"]["classification"] == "approved_direct" + assert fields["filing_dates"]["schema_status"] == "canonical_column_missing" + assert fields["filing_dates"]["required_owner_action"] == ( + "approve_canonical_schema_before_any_apply" + ) + assert "filing_dates" not in ticker["future_apply_candidate_fields"] + + def test_preview_blocks_mixed_candidate_periods(tmp_path: Path): payload = _companyfacts_payload(eps_period="2024-09-28") payload["facts"]["us-gaap"]["EarningsPerShareDiluted"]["units"][ @@ -507,7 +651,9 @@ def facts_fetcher(url, *_args): def test_preview_reports_staged_schema_expansion_and_is_deterministic(tmp_path: Path): - result, _ = _build(tmp_path) + retrieval_timestamp = "2026-08-18T22:15:30Z" + result, _ = _build(tmp_path, retrieval_timestamp=retrieval_timestamp) + rebuilt, _ = _build(tmp_path, retrieval_timestamp=retrieval_timestamp) assert result["schema_delta"]["staged_extra_columns"] == ["currency"] assert "cash_from_operations" in result["schema_delta"]["candidate_component_extra_columns"] @@ -515,4 +661,4 @@ def test_preview_reports_staged_schema_expansion_and_is_deterministic(tmp_path: assert "net_income" in result["schema_delta"]["canonical_columns_not_produced"] rendered = render_sec_fundamentals_preview(result) assert json.loads(rendered) == result - assert rendered == render_sec_fundamentals_preview(result) + assert rendered == render_sec_fundamentals_preview(rebuilt)