From a11baa042dac3cedcc958a8867d8ede86c2d8a24 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:54:31 -0500 Subject: [PATCH 01/15] fix: decode compressed SEC responses --- src/ledger/sec_client.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/ledger/sec_client.py b/src/ledger/sec_client.py index 1d0879f..f1a8bb4 100644 --- a/src/ledger/sec_client.py +++ b/src/ledger/sec_client.py @@ -1,7 +1,9 @@ from __future__ import annotations +import gzip import json import time +import zlib from pathlib import Path from typing import Any, Protocol, cast from urllib.error import HTTPError, URLError @@ -19,6 +21,20 @@ def _decode_payload(content: str, source: str) -> dict[str, Any]: return cast(dict[str, Any], payload) +def _decompress_body(content: bytes, content_encoding: str | None, source: str) -> bytes: + encoding = (content_encoding or "").strip().lower() + if encoding in ("", "identity"): + return content + if encoding == "gzip": + return gzip.decompress(content) + if encoding == "deflate": + try: + return zlib.decompress(content) + except zlib.error: + return zlib.decompress(content, -zlib.MAX_WBITS) + raise ValueError(f"Unsupported Content-Encoding {encoding!r} from {source}") + + class SecCompanyFactsClient: BASE_URL = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json" @@ -56,8 +72,20 @@ def fetch_companyfacts(self, cik: str) -> dict[str, Any]: try: self._last_request_at = time.monotonic() with urlopen(request, timeout=self.timeout_seconds) as response: # noqa: S310 - return _decode_payload(response.read().decode("utf-8"), url) - except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error: + body = _decompress_body( + response.read(), response.headers.get("Content-Encoding"), url + ) + return _decode_payload(body.decode("utf-8"), url) + except ( + HTTPError, + URLError, + TimeoutError, + json.JSONDecodeError, + UnicodeDecodeError, + OSError, + ValueError, + zlib.error, + ) as error: last_error = error if attempt >= self.max_retries: break From 0807c879e5fbfbd7d59934c10d07954df51b6c71 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:54:49 -0500 Subject: [PATCH 02/15] test: cover compressed SEC responses --- tests/test_sec_client.py | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/test_sec_client.py diff --git a/tests/test_sec_client.py b/tests/test_sec_client.py new file mode 100644 index 0000000..5dfee1d --- /dev/null +++ b/tests/test_sec_client.py @@ -0,0 +1,58 @@ +import gzip +import json +import zlib +from collections.abc import Callable +from types import TracebackType + +import pytest + +from ledger.sec_client import SecCompanyFactsClient + + +class FakeResponse: + def __init__(self, body: bytes, content_encoding: str) -> None: + self.body = body + self.headers = {"Content-Encoding": content_encoding} + + def read(self) -> bytes: + return self.body + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + return None + + +@pytest.mark.parametrize( + ("content_encoding", "compressor"), + [("gzip", gzip.compress), ("deflate", zlib.compress)], +) +def test_live_client_decodes_compressed_responses( + monkeypatch: pytest.MonkeyPatch, + content_encoding: str, + compressor: Callable[[bytes], bytes], +) -> None: + payload = {"cik": 320193, "entityName": "Example Device Company", "facts": {}} + response = FakeResponse( + compressor(json.dumps(payload).encode("utf-8")), + content_encoding, + ) + + def fake_urlopen(request: object, timeout: int) -> FakeResponse: + del request, timeout + return response + + monkeypatch.setattr("ledger.sec_client.urlopen", fake_urlopen) + client = SecCompanyFactsClient( + user_agent="Example Engineering data@example.org", + max_retries=0, + requests_per_second=1000, + ) + + assert client.fetch_companyfacts("320193") == payload From d5b52a889c664572e7f594ad8dc142a92d8309a9 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:55:02 -0500 Subject: [PATCH 03/15] fix: preserve financial fact period grain --- dbt/models/marts/fct_financial_facts.sql | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/dbt/models/marts/fct_financial_facts.sql b/dbt/models/marts/fct_financial_facts.sql index 88bbde7..9144e1c 100644 --- a/dbt/models/marts/fct_financial_facts.sql +++ b/dbt/models/marts/fct_financial_facts.sql @@ -6,15 +6,32 @@ deduplicated as ( select *, row_number() over ( - partition by cik, fact_namespace, fact_name, unit, accession_number, period_end_date + partition by + cik, + fact_namespace, + fact_name, + unit, + accession_number, + period_start_date, + period_end_date order by filed_date desc ) as row_number from facts ) select - md5(concat_ws('|', cik, fact_namespace, fact_name, unit, accession_number, period_end_date)) - as financial_fact_id, + md5( + concat_ws( + '|', + cik, + fact_namespace, + fact_name, + unit, + accession_number, + coalesce(cast(period_start_date as varchar), '__instant__'), + cast(period_end_date as varchar) + ) + ) as financial_fact_id, cik, entity_name, fact_namespace, From 9c4f9482ca8381e8d5db0c3d24ca25e58dc9c2f2 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:55:15 -0500 Subject: [PATCH 04/15] test: add distinct duration observations --- tests/fixtures/CIK0000320193.json | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/CIK0000320193.json b/tests/fixtures/CIK0000320193.json index f6a3583..43afd89 100644 --- a/tests/fixtures/CIK0000320193.json +++ b/tests/fixtures/CIK0000320193.json @@ -9,15 +9,26 @@ "units": { "USD": [ { - "val": 1000000, + "val": 250000, "accn": "0000000000-26-000001", "fy": 2026, - "fp": "FY", - "form": "10-K", - "filed": "2026-01-31", - "start": "2025-01-01", - "end": "2025-12-31", - "frame": "CY2025" + "fp": "Q3", + "form": "10-Q", + "filed": "2026-10-31", + "start": "2026-07-01", + "end": "2026-09-30", + "frame": "CY2026Q3" + }, + { + "val": 750000, + "accn": "0000000000-26-000001", + "fy": 2026, + "fp": "Q3", + "form": "10-Q", + "filed": "2026-10-31", + "start": "2026-01-01", + "end": "2026-09-30", + "frame": "CY2026Q3YTD" } ] } From 2ccf1b32e79811d424fbbb172abfbb2095347737 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:55:32 -0500 Subject: [PATCH 05/15] test: verify distinct reporting periods survive normalization --- tests/test_pipeline.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index cfc7eb4..6392be4 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -8,14 +8,17 @@ FIXTURE_DIR = Path(__file__).parent / "fixtures" -def test_normalization_extracts_financial_observations() -> None: +def test_normalization_extracts_distinct_financial_periods() -> None: payload = json.loads((FIXTURE_DIR / "CIK0000320193.json").read_text(encoding="utf-8")) rows = normalize_companyfacts(payload) - assert len(rows) == 1 - assert rows[0]["cik"] == "0000320193" - assert rows[0]["fact_name"] == "RevenueFromContractWithCustomerExcludingAssessedTax" - assert rows[0]["value"] == 1000000 + assert len(rows) == 2 + assert {row["cik"] for row in rows} == {"0000320193"} + assert {row["fact_name"] for row in rows} == { + "RevenueFromContractWithCustomerExcludingAssessedTax" + } + assert {row["period_start_date"] for row in rows} == {"2026-01-01", "2026-07-01"} + assert {row["value"] for row in rows} == {250000, 750000} def test_pipeline_is_repeatable_and_writes_governed_outputs(tmp_path: Path) -> None: @@ -34,8 +37,8 @@ def test_pipeline_is_repeatable_and_writes_governed_outputs(tmp_path: Path) -> N second = run_pipeline(config, client) assert first.issuer_count == 2 - assert first.normalized_fact_count == 2 - assert second.normalized_fact_count == 2 + assert first.normalized_fact_count == 3 + assert second.normalized_fact_count == 3 rows = [ json.loads(line) for line in (tmp_path / "normalized" / "financial_facts.jsonl") From 18540ebe8bc62186c6c78ff30da22e680c63667c Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:55:47 -0500 Subject: [PATCH 06/15] test: protect financial fact period grain --- ...financial_facts_preserves_period_grain.sql | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 dbt/tests/test_fct_financial_facts_preserves_period_grain.sql diff --git a/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql b/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql new file mode 100644 index 0000000..03b8a8d --- /dev/null +++ b/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql @@ -0,0 +1,35 @@ +with source_periods as ( + select + cik, + fact_namespace, + fact_name, + unit, + accession_number, + period_end_date, + count( + distinct coalesce(cast(period_start_date as varchar), '__instant__') + ) as period_count + from {{ ref('stg_financial_facts') }} + group by all +), + +mart_periods as ( + select + cik, + fact_namespace, + fact_name, + unit, + accession_number, + period_end_date, + count( + distinct coalesce(cast(period_start_date as varchar), '__instant__') + ) as period_count + from {{ ref('fct_financial_facts') }} + group by all +) + +select source_periods.* +from source_periods +left join mart_periods + using (cik, fact_namespace, fact_name, unit, accession_number, period_end_date) +where source_periods.period_count != coalesce(mart_periods.period_count, 0) From 954e712d25fbc015f677db16c4fa14b0d7b5ea57 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:56:07 -0500 Subject: [PATCH 07/15] ci: enforce Ruff formatting --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a2d0fe..9baf59c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,8 @@ jobs: name: ruff-diagnostics path: ruff-output.txt retention-days: 7 + - name: Ruff format + run: uv run ruff format --check src tests - name: Mypy run: uv run mypy src - name: Pytest From 1f038e9f7543540fe1da396419afbb0d54603da8 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:56:14 -0500 Subject: [PATCH 08/15] ci: align local validation with formatting gate --- scripts/validate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/validate.sh b/scripts/validate.sh index 57bb5ae..e127d5d 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -2,6 +2,7 @@ set -euo pipefail uv run ruff check src tests +uv run ruff format --check src tests uv run mypy src uv run pytest uv run dbt parse --project-dir dbt --profiles-dir dbt From f52b6c77793a20ace94e363129fb2a0f72274e7f Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:57:05 -0500 Subject: [PATCH 09/15] style: normalize compressed-response test annotations --- tests/test_sec_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_sec_client.py b/tests/test_sec_client.py index 5dfee1d..746a52c 100644 --- a/tests/test_sec_client.py +++ b/tests/test_sec_client.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import gzip import json import zlib @@ -17,7 +19,7 @@ def __init__(self, body: bytes, content_encoding: str) -> None: def read(self) -> bytes: return self.body - def __enter__(self) -> "FakeResponse": + def __enter__(self) -> FakeResponse: return self def __exit__( From bded1f902bdb19d1626f631b643b2f5142ca1c47 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:57:28 -0500 Subject: [PATCH 10/15] fix: retry truncated compressed responses --- src/ledger/sec_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ledger/sec_client.py b/src/ledger/sec_client.py index f1a8bb4..3af9f33 100644 --- a/src/ledger/sec_client.py +++ b/src/ledger/sec_client.py @@ -77,6 +77,7 @@ def fetch_companyfacts(self, cik: str) -> dict[str, Any]: ) return _decode_payload(body.decode("utf-8"), url) except ( + EOFError, HTTPError, URLError, TimeoutError, From 87f5d582180faf82d07303f68da05da741aa6ecf Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:00:13 -0500 Subject: [PATCH 11/15] ci: preserve Ruff format diagnostics --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9baf59c..6207fd4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,16 @@ jobs: path: ruff-output.txt retention-days: 7 - name: Ruff format - run: uv run ruff format --check src tests + run: | + set -o pipefail + uv run ruff format --check --diff src tests 2>&1 | tee ruff-format-output.txt + - name: Upload Ruff format diagnostics + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ruff-format-diagnostics + path: ruff-format-output.txt + retention-days: 7 - name: Mypy run: uv run mypy src - name: Pytest From 33a81b32f0e27731378478e54a8327e78d6ffe73 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:01:09 -0500 Subject: [PATCH 12/15] style: apply Ruff formatting --- src/ledger/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ledger/config.py b/src/ledger/config.py index c103501..bcd9499 100644 --- a/src/ledger/config.py +++ b/src/ledger/config.py @@ -94,7 +94,5 @@ def load_config(path: str | Path, environ: Mapping[str, str] | None = None) -> L _env_value(environment, pipeline, "request_timeout_seconds_env", 30) ), max_retries=int(_env_value(environment, pipeline, "max_retries_env", 3)), - requests_per_second=float( - _env_value(environment, pipeline, "requests_per_second_env", 5) - ), + requests_per_second=float(_env_value(environment, pipeline, "requests_per_second_env", 5)), ) From 6dbaa1b62799fb861e86f64f17b6f141d265c3d6 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:03:05 -0500 Subject: [PATCH 13/15] test: preserve period and value associations --- tests/test_pipeline.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6392be4..289c186 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -17,8 +17,10 @@ def test_normalization_extracts_distinct_financial_periods() -> None: assert {row["fact_name"] for row in rows} == { "RevenueFromContractWithCustomerExcludingAssessedTax" } - assert {row["period_start_date"] for row in rows} == {"2026-01-01", "2026-07-01"} - assert {row["value"] for row in rows} == {250000, 750000} + assert {(row["period_start_date"], row["value"]) for row in rows} == { + ("2026-01-01", 750000), + ("2026-07-01", 250000), + } def test_pipeline_is_repeatable_and_writes_governed_outputs(tmp_path: Path) -> None: From 816d47d103f354112c06428afeda8fbbefd3c69f Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:03:25 -0500 Subject: [PATCH 14/15] test: compare exact financial period identities --- ...financial_facts_preserves_period_grain.sql | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql b/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql index 03b8a8d..e8dce2a 100644 --- a/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql +++ b/dbt/tests/test_fct_financial_facts_preserves_period_grain.sql @@ -1,35 +1,39 @@ with source_periods as ( - select + select distinct cik, fact_namespace, fact_name, unit, accession_number, period_end_date, - count( - distinct coalesce(cast(period_start_date as varchar), '__instant__') - ) as period_count + coalesce(cast(period_start_date as varchar), '__instant__') as period_start_key from {{ ref('stg_financial_facts') }} - group by all ), mart_periods as ( - select + select distinct cik, fact_namespace, fact_name, unit, accession_number, period_end_date, - count( - distinct coalesce(cast(period_start_date as varchar), '__instant__') - ) as period_count + coalesce(cast(period_start_date as varchar), '__instant__') as period_start_key from {{ ref('fct_financial_facts') }} - group by all +), + +missing_from_mart as ( + select * from source_periods + except + select * from mart_periods +), + +unexpected_in_mart as ( + select * from mart_periods + except + select * from source_periods ) -select source_periods.* -from source_periods -left join mart_periods - using (cik, fact_namespace, fact_name, unit, accession_number, period_end_date) -where source_periods.period_count != coalesce(mart_periods.period_count, 0) +select * from missing_from_mart +union all +select * from unexpected_in_mart From d00e5c2679a2bd73d64475667991fa1aa09cf452 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:03:57 -0500 Subject: [PATCH 15/15] test: cover truncated compressed response retries --- tests/test_sec_client.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_sec_client.py b/tests/test_sec_client.py index 746a52c..50fb0fe 100644 --- a/tests/test_sec_client.py +++ b/tests/test_sec_client.py @@ -58,3 +58,40 @@ def fake_urlopen(request: object, timeout: int) -> FakeResponse: ) assert client.fetch_companyfacts("320193") == payload + + +@pytest.mark.parametrize( + ("content_encoding", "compressor"), + [("gzip", gzip.compress), ("deflate", zlib.compress)], +) +def test_live_client_retries_truncated_compressed_responses( + monkeypatch: pytest.MonkeyPatch, + content_encoding: str, + compressor: Callable[[bytes], bytes], +) -> None: + payload = {"cik": 320193, "entityName": "Example Device Company", "facts": {}} + compressed = compressor(json.dumps(payload).encode("utf-8")) + responses = iter( + [ + FakeResponse(compressed[: len(compressed) // 2], content_encoding), + FakeResponse(compressed, content_encoding), + ] + ) + call_count = 0 + + def fake_urlopen(request: object, timeout: int) -> FakeResponse: + nonlocal call_count + del request, timeout + call_count += 1 + return next(responses) + + monkeypatch.setattr("ledger.sec_client.urlopen", fake_urlopen) + monkeypatch.setattr("ledger.sec_client.time.sleep", lambda _seconds: None) + client = SecCompanyFactsClient( + user_agent="Example Engineering data@example.org", + max_retries=1, + requests_per_second=1000, + ) + + assert client.fetch_companyfacts("320193") == payload + assert call_count == 2