From a33e56ab738999df80249c9aba08e77a3591d254 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:35:02 -0500 Subject: [PATCH 01/15] add reusable project template foundation --- .dockerignore | 9 ++++ .env.example | 8 ++++ .gitignore | 18 +++++++ CONTRIBUTING.md | 16 +++++++ Dockerfile | 16 +++++++ Makefile | 17 +++++++ README.md | 99 +++++++++++++++++++++++++++++++++++++++ config/ledger.example.yml | 20 ++++++++ docs/ARCHITECTURE.md | 26 ++++++++++ docs/CUSTOMIZATION.md | 35 ++++++++++++++ pyproject.toml | 46 ++++++++++++++++++ scripts/validate.sh | 7 +++ 12 files changed, 317 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 config/ledger.example.yml create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/CUSTOMIZATION.md create mode 100644 pyproject.toml create mode 100755 scripts/validate.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..27f0b0b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.github +.venv +__pycache__ +data +logs +tests +infra +dbt/target diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..73d7082 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# Required only for live SEC requests. +LEDGER_SEC_USER_AGENT=YourCompany Data Engineering your-team@example.com + +# Optional overrides. +LEDGER_OUTPUT_DIR=data +LEDGER_REQUEST_TIMEOUT_SECONDS=30 +LEDGER_MAX_RETRIES=3 +LEDGER_REQUESTS_PER_SECOND=5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aced8b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.env +config/ledger.yml +data/ +logs/ +target/ +dbt/target/ +dbt/logs/ +*.duckdb +.terraform/ +*.tfstate +*.tfstate.* +.DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4d75a84 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing + +Create a focused branch, add or update tests with each behavior change, and run: + +```bash +bash scripts/validate.sh +``` + +Pull requests should state: +- the observable behavior changed +- the data grain and contract affected +- fixture coverage +- full-refresh and repeat-run behavior +- deployment or migration implications + +Do not include personal identifiers, private infrastructure names, credentials, or generated runtime evidence in commits. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..83ef404 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_COMPILE_BYTECODE=1 + +WORKDIR /app + +RUN pip install --no-cache-dir uv +COPY pyproject.toml README.md ./ +COPY src ./src +COPY config ./config +RUN uv sync --no-dev + +ENTRYPOINT ["uv", "run", "python", "-m", "ledger.cli"] +CMD ["--help"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4c759e7 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +.PHONY: install test validate fixture dbt + +install: + uv sync --group dev + +test: + uv run pytest + +validate: + bash scripts/validate.sh + +fixture: + cp -n config/ledger.example.yml config/ledger.yml || true + uv run python -m ledger.cli --config config/ledger.yml --fixture-dir tests/fixtures + +dbt: + uv run dbt build --project-dir dbt --profiles-dir dbt diff --git a/README.md b/README.md new file mode 100644 index 0000000..8a7a5f8 --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# Project Ledger + +Project Ledger is a reusable reference implementation for building a production-oriented SEC/XBRL data pipeline. It demonstrates fixture-first development, governed issuer selection, idempotent ingestion, normalized financial facts, local DuckDB/dbt modeling, optional Airflow orchestration, and a parameterized GCP deployment path. + +The repository is intentionally organization-neutral. It contains no personal identifiers, private infrastructure names, fixed cloud project IDs, or embedded credentials. + +## Architecture + +```text +SEC Company Facts API or fixtures + | + v +Python ingestion and validation + | + +----+--------------------+ + | | + v v +raw JSON per issuer normalized JSONL facts + | | + +------------+------------+ + v + dbt + DuckDB + | + v + analytical marts +``` + +Optional production path: + +```text +GitHub Actions -> container image -> Cloud Run Job -> GCS/BigQuery + ^ + | + Airflow/Composer +``` + +## Quick start + +1. Install Python 3.12 and `uv`. +2. Copy the examples: + +```bash +cp .env.example .env +cp config/ledger.example.yml config/ledger.yml +``` + +3. Set a descriptive SEC user agent in `.env`: + +```text +LEDGER_SEC_USER_AGENT=YourCompany Data Engineering your-team@example.com +``` + +4. Install and validate: + +```bash +uv sync --group dev +uv run pytest +bash scripts/validate.sh +``` + +5. Run from deterministic fixtures: + +```bash +uv run python -m ledger.cli \ + --config config/ledger.yml \ + --fixture-dir tests/fixtures +``` + +6. Run against the live SEC API only after configuring a valid user agent: + +```bash +uv run python -m ledger.cli --config config/ledger.yml --live +``` + +7. Build the local warehouse: + +```bash +uv run dbt build --project-dir dbt --profiles-dir dbt +``` + +## Template boundaries + +Included: +- Python ingestion and normalization +- fixture-based tests +- dbt/DuckDB models +- CI validation +- Airflow DAG example +- Docker packaging +- parameterized GCP Terraform skeleton +- architecture and customization guidance + +Deliberately excluded: +- personal sprint journals and validation history +- generated dashboard exports and runtime evidence +- real cloud project IDs, bucket names, service accounts, emails, and secrets +- issuer selections tied to a specific individual or organization + +See `docs/CUSTOMIZATION.md` before deploying. diff --git a/config/ledger.example.yml b/config/ledger.example.yml new file mode 100644 index 0000000..b34a457 --- /dev/null +++ b/config/ledger.example.yml @@ -0,0 +1,20 @@ +project: + name: project-ledger + +sec: + # The CLI reads the actual value from LEDGER_SEC_USER_AGENT. + user_agent_env: LEDGER_SEC_USER_AGENT + issuers: + - symbol: AAPL + cik: "0000320193" + - symbol: MSFT + cik: "0000789019" + +storage: + output_dir_env: LEDGER_OUTPUT_DIR + default_output_dir: data + +pipeline: + request_timeout_seconds_env: LEDGER_REQUEST_TIMEOUT_SECONDS + max_retries_env: LEDGER_MAX_RETRIES + requests_per_second_env: LEDGER_REQUESTS_PER_SECOND diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..9855886 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,26 @@ +# Architecture + +## Design goals + +1. **Determinism before live integration.** Every transformation can be exercised from committed fixtures. +2. **Configuration over identity.** Issuers, project names, output paths, cloud projects, and user agents are parameters. +3. **Raw preservation before normalization.** Original payloads are retained before analytical rows are produced. +4. **Idempotent outputs.** Re-running the same inputs replaces governed outputs rather than appending duplicates. +5. **Local-to-cloud parity.** Python and dbt contracts remain stable when storage and orchestration are replaced. + +## Components + +| Component | Responsibility | +| --- | --- | +| `ledger.config` | Parse and validate environment-neutral settings | +| `ledger.sec_client` | Retrieve live Company Facts payloads or deterministic fixtures | +| `ledger.pipeline` | Validate issuer identity and normalize XBRL observations | +| `ledger.storage` | Atomic local writes for raw JSON and normalized JSONL | +| `dbt/` | Stage and deduplicate analytical facts in DuckDB | +| `dags/` | Example Airflow control plane | +| `infra/gcp/` | Parameterized deployment skeleton | +| `.github/workflows/` | Repeatable validation and smoke testing | + +## Production extension points + +Replace local storage behind a stable interface with GCS and BigQuery loaders. Add schema contracts before promotion. Add observability around freshness, row counts, failed issuer fetches, and cost. Store the SEC user agent in Secret Manager. Keep fixtures as a permanent CI lane even after live infrastructure exists. diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md new file mode 100644 index 0000000..042dbec --- /dev/null +++ b/docs/CUSTOMIZATION.md @@ -0,0 +1,35 @@ +# Customizing the template + +## Required changes + +1. Replace the example issuer allowlist in `config/ledger.example.yml`. +2. Set `LEDGER_SEC_USER_AGENT` to a monitored organizational contact. +3. Decide whether local DuckDB is sufficient or BigQuery is required. +4. Rename Terraform resources only through variables or `locals`; do not hard-code personal names. +5. Define retention, data classification, and access policies before storing filing documents or derived text. + +## Reusing the architecture outside SEC data + +The reusable pattern is: + +```text +allowlisted entities -> fixture/live client -> raw preservation -> normalization -> dbt contracts -> governed marts +``` + +To adapt it: +- implement another client with the same fetch boundary +- replace `normalize_companyfacts` with domain-specific normalization +- preserve fixture parity +- keep stable raw and normalized contracts +- update dbt models and tests at the same grain + +## Security rules + +Never commit: +- cloud project IDs tied to private environments +- service-account keys +- personal email addresses or phone numbers +- API keys, cookies, tokens, or secrets +- production query results or unredacted logs + +Use workload identity for CI and Secret Manager for runtime credentials. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f5692c8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "project-ledger-template" +version = "0.1.0" +description = "Reusable SEC/XBRL data engineering reference pipeline" +readme = "README.md" +requires-python = ">=3.12,<3.13" +dependencies = [ + "pyyaml>=6.0.2", +] + +[dependency-groups] +dev = [ + "pytest>=8.3,<10", + "ruff>=0.9,<1", + "mypy>=1.14,<2", + "types-PyYAML>=6.0.12", + "dbt-core>=1.9,<2", + "dbt-duckdb>=1.9,<2", +] +airflow = [ + "apache-airflow>=2.10,<4", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/ledger"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] + +[tool.mypy] +python_version = "3.12" +mypy_path = "src" +packages = ["ledger"] +strict = true + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] diff --git a/scripts/validate.sh b/scripts/validate.sh new file mode 100755 index 0000000..57bb5ae --- /dev/null +++ b/scripts/validate.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +uv run ruff check src tests +uv run mypy src +uv run pytest +uv run dbt parse --project-dir dbt --profiles-dir dbt From 9f38a9aa85a2ccb2423252558769fafdd2ad6dae Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:36:31 -0500 Subject: [PATCH 02/15] add configurable ingestion pipeline and tests --- src/ledger/__init__.py | 6 ++ src/ledger/cli.py | 43 ++++++++++++++ src/ledger/config.py | 99 +++++++++++++++++++++++++++++++ src/ledger/pipeline.py | 84 ++++++++++++++++++++++++++ src/ledger/sec_client.py | 70 ++++++++++++++++++++++ src/ledger/storage.py | 38 ++++++++++++ tests/fixtures/CIK0000320193.json | 27 +++++++++ tests/fixtures/CIK0000789019.json | 27 +++++++++ tests/test_config.py | 50 ++++++++++++++++ tests/test_pipeline.py | 46 ++++++++++++++ 10 files changed, 490 insertions(+) create mode 100644 src/ledger/__init__.py create mode 100644 src/ledger/cli.py create mode 100644 src/ledger/config.py create mode 100644 src/ledger/pipeline.py create mode 100644 src/ledger/sec_client.py create mode 100644 src/ledger/storage.py create mode 100644 tests/fixtures/CIK0000320193.json create mode 100644 tests/fixtures/CIK0000789019.json create mode 100644 tests/test_config.py create mode 100644 tests/test_pipeline.py diff --git a/src/ledger/__init__.py b/src/ledger/__init__.py new file mode 100644 index 0000000..880f581 --- /dev/null +++ b/src/ledger/__init__.py @@ -0,0 +1,6 @@ +"""Project Ledger reusable pipeline package.""" + +from ledger.config import Issuer, LedgerConfig, load_config +from ledger.pipeline import PipelineSummary, run_pipeline + +__all__ = ["Issuer", "LedgerConfig", "PipelineSummary", "load_config", "run_pipeline"] diff --git a/src/ledger/cli.py b/src/ledger/cli.py new file mode 100644 index 0000000..faa4449 --- /dev/null +++ b/src/ledger/cli.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from ledger.config import load_config +from ledger.pipeline import run_pipeline +from ledger.sec_client import FixtureCompanyFactsClient, SecCompanyFactsClient + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the Project Ledger SEC ingestion pipeline") + parser.add_argument("--config", default="config/ledger.yml", help="Path to YAML config") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--fixture-dir", help="Read deterministic SEC fixtures from a directory") + source.add_argument("--live", action="store_true", help="Call the live SEC Company Facts API") + return parser + + +def main() -> int: + args = build_parser().parse_args() + config = load_config(Path(args.config)) + + if args.live: + config.require_live_access() + assert config.sec_user_agent is not None + client = SecCompanyFactsClient( + user_agent=config.sec_user_agent, + timeout_seconds=config.request_timeout_seconds, + max_retries=config.max_retries, + requests_per_second=config.requests_per_second, + ) + else: + client = FixtureCompanyFactsClient(args.fixture_dir) + + summary = run_pipeline(config, client) + print(json.dumps(summary.to_dict(), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ledger/config.py b/src/ledger/config.py new file mode 100644 index 0000000..c27d88b --- /dev/null +++ b/src/ledger/config.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +import yaml + + +class ConfigurationError(ValueError): + """Raised when configuration is missing or unsafe.""" + + +@dataclass(frozen=True) +class Issuer: + symbol: str + cik: str + + def __post_init__(self) -> None: + normalized = self.cik.strip().zfill(10) + if not normalized.isdigit() or len(normalized) != 10: + raise ConfigurationError(f"Invalid CIK for {self.symbol!r}: {self.cik!r}") + object.__setattr__(self, "cik", normalized) + object.__setattr__(self, "symbol", self.symbol.strip().upper()) + + +@dataclass(frozen=True) +class LedgerConfig: + project_name: str + sec_user_agent: str | None + issuers: tuple[Issuer, ...] + output_dir: Path + request_timeout_seconds: int + max_retries: int + requests_per_second: float + + def require_live_access(self) -> None: + value = (self.sec_user_agent or "").strip() + if not value or "example.com" in value.lower() or "yourcompany" in value.lower(): + raise ConfigurationError( + "Live SEC requests require LEDGER_SEC_USER_AGENT with an organization and " + "monitored contact address." + ) + + +def _env_value( + environ: Mapping[str, str], + section: Mapping[str, Any], + env_key_field: str, + default: Any, +) -> Any: + env_name = section.get(env_key_field) + if env_name and environ.get(str(env_name)) not in (None, ""): + return environ[str(env_name)] + return default + + +def load_config(path: str | Path, environ: Mapping[str, str] | None = None) -> LedgerConfig: + environment = os.environ if environ is None else environ + config_path = Path(path) + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + + project = raw.get("project", {}) + sec = raw.get("sec", {}) + storage = raw.get("storage", {}) + pipeline = raw.get("pipeline", {}) + + issuer_rows = sec.get("issuers", []) + issuers = tuple(Issuer(symbol=row["symbol"], cik=str(row["cik"])) for row in issuer_rows) + if not issuers: + raise ConfigurationError("At least one issuer must be configured.") + if len({issuer.cik for issuer in issuers}) != len(issuers): + raise ConfigurationError("Issuer CIK values must be unique.") + + output_dir = Path( + str( + _env_value( + environment, + storage, + "output_dir_env", + storage.get("default_output_dir", "data"), + ) + ) + ) + + return LedgerConfig( + project_name=str(project.get("name", "project-ledger")), + sec_user_agent=environment.get(str(sec.get("user_agent_env", "LEDGER_SEC_USER_AGENT"))), + issuers=issuers, + output_dir=output_dir, + request_timeout_seconds=int( + _env_value(environment, pipeline, "request_timeout_seconds_env", 30) + ), + max_retries=int(_env_value(environment, pipeline, "max_retries_env", 3)), + requests_per_second=float( + _env_value(environment, pipeline, "requests_per_second_env", 5) + ), + ) diff --git a/src/ledger/pipeline.py b/src/ledger/pipeline.py new file mode 100644 index 0000000..85f3a28 --- /dev/null +++ b/src/ledger/pipeline.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from typing import Any + +from ledger.config import LedgerConfig +from ledger.sec_client import CompanyFactsClient +from ledger.storage import write_json, write_jsonl + + +@dataclass(frozen=True) +class PipelineSummary: + run_timestamp: str + issuer_count: int + normalized_fact_count: int + raw_files: tuple[str, ...] + normalized_file: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def normalize_companyfacts(payload: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + cik = str(payload.get("cik", "")).zfill(10) + entity_name = payload.get("entityName") + + for namespace, facts in (payload.get("facts") or {}).items(): + for fact_name, fact_definition in facts.items(): + units = fact_definition.get("units") or {} + for unit_name, observations in units.items(): + for observation in observations: + rows.append( + { + "cik": cik, + "entity_name": entity_name, + "namespace": namespace, + "fact_name": fact_name, + "label": fact_definition.get("label"), + "description": fact_definition.get("description"), + "unit": unit_name, + "value": observation.get("val"), + "accession_number": observation.get("accn"), + "fiscal_year": observation.get("fy"), + "fiscal_period": observation.get("fp"), + "form": observation.get("form"), + "filed_date": observation.get("filed"), + "frame": observation.get("frame"), + "period_start_date": observation.get("start"), + "period_end_date": observation.get("end"), + } + ) + return rows + + +def run_pipeline(config: LedgerConfig, client: CompanyFactsClient) -> PipelineSummary: + raw_directory = config.output_dir / "raw" / "companyfacts" + normalized_path = config.output_dir / "normalized" / "financial_facts.jsonl" + + raw_files: list[str] = [] + normalized_rows: list[dict[str, Any]] = [] + + for issuer in config.issuers: + payload = client.fetch_companyfacts(issuer.cik) + payload_cik = str(payload.get("cik", "")).zfill(10) + if payload_cik != issuer.cik: + raise ValueError( + f"CIK mismatch for {issuer.symbol}: expected {issuer.cik}, received {payload_cik}" + ) + + raw_path = raw_directory / f"CIK{issuer.cik}.json" + write_json(raw_path, payload) + raw_files.append(str(raw_path)) + normalized_rows.extend(normalize_companyfacts(payload)) + + normalized_count = write_jsonl(normalized_path, normalized_rows) + return PipelineSummary( + run_timestamp=datetime.now(UTC).isoformat(), + issuer_count=len(config.issuers), + normalized_fact_count=normalized_count, + raw_files=tuple(raw_files), + normalized_file=str(normalized_path), + ) diff --git a/src/ledger/sec_client.py b/src/ledger/sec_client.py new file mode 100644 index 0000000..1b973f2 --- /dev/null +++ b/src/ledger/sec_client.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any, Protocol +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +class CompanyFactsClient(Protocol): + def fetch_companyfacts(self, cik: str) -> dict[str, Any]: ... + + +class SecCompanyFactsClient: + BASE_URL = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json" + + def __init__( + self, + *, + user_agent: str, + timeout_seconds: int = 30, + max_retries: int = 3, + requests_per_second: float = 5, + ) -> None: + self.user_agent = user_agent + self.timeout_seconds = timeout_seconds + self.max_retries = max_retries + self.minimum_interval = 1.0 / max(requests_per_second, 0.1) + self._last_request_at = 0.0 + + def fetch_companyfacts(self, cik: str) -> dict[str, Any]: + url = self.BASE_URL.format(cik=cik.zfill(10)) + last_error: Exception | None = None + + for attempt in range(self.max_retries + 1): + elapsed = time.monotonic() - self._last_request_at + if elapsed < self.minimum_interval: + time.sleep(self.minimum_interval - elapsed) + + request = Request( + url, + headers={ + "User-Agent": self.user_agent, + "Accept-Encoding": "gzip, deflate", + "Host": "data.sec.gov", + }, + ) + try: + self._last_request_at = time.monotonic() + with urlopen(request, timeout=self.timeout_seconds) as response: # noqa: S310 + return json.loads(response.read().decode("utf-8")) + except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error: + last_error = error + if attempt >= self.max_retries: + break + time.sleep(min(2**attempt, 8)) + + raise RuntimeError(f"Unable to fetch SEC company facts for CIK {cik}") from last_error + + +class FixtureCompanyFactsClient: + def __init__(self, fixture_dir: str | Path) -> None: + self.fixture_dir = Path(fixture_dir) + + def fetch_companyfacts(self, cik: str) -> dict[str, Any]: + fixture_path = self.fixture_dir / f"CIK{cik.zfill(10)}.json" + if not fixture_path.exists(): + raise FileNotFoundError(f"Missing fixture: {fixture_path}") + return json.loads(fixture_path.read_text(encoding="utf-8")) diff --git a/src/ledger/storage.py b/src/ledger/storage.py new file mode 100644 index 0000000..29d2a0c --- /dev/null +++ b/src/ledger/storage.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.") + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, path) + except Exception: + Path(temporary_name).unlink(missing_ok=True) + raise + + +def write_json(path: Path, payload: Mapping[str, Any]) -> str: + content = json.dumps(payload, indent=2, sort_keys=True) + "\n" + _atomic_write(path, content) + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def write_jsonl(path: Path, rows: Iterable[Mapping[str, Any]]) -> int: + materialized = [json.dumps(row, sort_keys=True) for row in rows] + content = "\n".join(materialized) + if content: + content += "\n" + _atomic_write(path, content) + return len(materialized) diff --git a/tests/fixtures/CIK0000320193.json b/tests/fixtures/CIK0000320193.json new file mode 100644 index 0000000..f6a3583 --- /dev/null +++ b/tests/fixtures/CIK0000320193.json @@ -0,0 +1,27 @@ +{ + "cik": 320193, + "entityName": "Example Device Company", + "facts": { + "us-gaap": { + "RevenueFromContractWithCustomerExcludingAssessedTax": { + "label": "Revenue", + "description": "Illustrative revenue fixture.", + "units": { + "USD": [ + { + "val": 1000000, + "accn": "0000000000-26-000001", + "fy": 2026, + "fp": "FY", + "form": "10-K", + "filed": "2026-01-31", + "start": "2025-01-01", + "end": "2025-12-31", + "frame": "CY2025" + } + ] + } + } + } + } +} diff --git a/tests/fixtures/CIK0000789019.json b/tests/fixtures/CIK0000789019.json new file mode 100644 index 0000000..5b24901 --- /dev/null +++ b/tests/fixtures/CIK0000789019.json @@ -0,0 +1,27 @@ +{ + "cik": 789019, + "entityName": "Example Software Company", + "facts": { + "us-gaap": { + "NetIncomeLoss": { + "label": "Net income", + "description": "Illustrative net-income fixture.", + "units": { + "USD": [ + { + "val": 250000, + "accn": "0000000000-26-000002", + "fy": 2026, + "fp": "FY", + "form": "10-K", + "filed": "2026-02-15", + "start": "2025-01-01", + "end": "2025-12-31", + "frame": "CY2025" + } + ] + } + } + } + } +} diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..aa37c7c --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import pytest + +from ledger.config import ConfigurationError, load_config + + +CONFIG = """ +project: + name: template-test +sec: + user_agent_env: LEDGER_SEC_USER_AGENT + issuers: + - symbol: test + cik: "320193" +storage: + output_dir_env: LEDGER_OUTPUT_DIR + default_output_dir: data +pipeline: + request_timeout_seconds_env: LEDGER_REQUEST_TIMEOUT_SECONDS + max_retries_env: LEDGER_MAX_RETRIES + requests_per_second_env: LEDGER_REQUESTS_PER_SECOND +""" + + +def test_load_config_normalizes_values(tmp_path: Path) -> None: + path = tmp_path / "ledger.yml" + path.write_text(CONFIG, encoding="utf-8") + config = load_config( + path, + { + "LEDGER_SEC_USER_AGENT": "Example Engineering data@example.org", + "LEDGER_OUTPUT_DIR": str(tmp_path / "out"), + "LEDGER_MAX_RETRIES": "4", + }, + ) + + assert config.issuers[0].symbol == "TEST" + assert config.issuers[0].cik == "0000320193" + assert config.output_dir == tmp_path / "out" + assert config.max_retries == 4 + + +def test_live_access_rejects_placeholder_user_agent(tmp_path: Path) -> None: + path = tmp_path / "ledger.yml" + path.write_text(CONFIG, encoding="utf-8") + config = load_config(path, {"LEDGER_SEC_USER_AGENT": "YourCompany team@example.com"}) + + with pytest.raises(ConfigurationError): + config.require_live_access() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..9b30e99 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,46 @@ +import json +from pathlib import Path + +from ledger.config import Issuer, LedgerConfig +from ledger.pipeline import normalize_companyfacts, run_pipeline +from ledger.sec_client import FixtureCompanyFactsClient + + +FIXTURE_DIR = Path(__file__).parent / "fixtures" + + +def test_normalization_extracts_financial_observations() -> None: + payload = json.loads((FIXTURE_DIR / "CIK0000320193.json").read_text(encoding="utf-8")) + rows = normalize_companyfacts(payload) + + assert len(rows) == 1 + assert rows[0]["cik"] == "0000320193" + assert rows[0]["fact_name"] == "RevenueFromContractWithCustomerExcludingAssessedTax" + assert rows[0]["value"] == 1000000 + + +def test_pipeline_is_repeatable_and_writes_governed_outputs(tmp_path: Path) -> None: + config = LedgerConfig( + project_name="test", + sec_user_agent=None, + issuers=(Issuer("AAPL", "320193"), Issuer("MSFT", "789019")), + output_dir=tmp_path, + request_timeout_seconds=30, + max_retries=1, + requests_per_second=5, + ) + client = FixtureCompanyFactsClient(FIXTURE_DIR) + + first = run_pipeline(config, client) + second = run_pipeline(config, client) + + assert first.issuer_count == 2 + assert first.normalized_fact_count == 2 + assert second.normalized_fact_count == 2 + rows = [ + json.loads(line) + for line in (tmp_path / "normalized" / "financial_facts.jsonl") + .read_text(encoding="utf-8") + .splitlines() + ] + assert {row["cik"] for row in rows} == {"0000320193", "0000789019"} From 971c5e599ea60f2e82e01bfede7e01ead0450b4b Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:37:25 -0500 Subject: [PATCH 03/15] add dbt orchestration CI and GCP deployment skeleton --- .github/workflows/ci.yml | 27 +++++ dags/ledger_daily_pipeline.py | 37 +++++++ dbt/dbt_project.yml | 20 ++++ dbt/models/marts/fct_financial_facts.sql | 35 +++++++ dbt/models/schema.yml | 25 +++++ dbt/models/staging/stg_financial_facts.sql | 23 +++++ dbt/profiles.yml | 7 ++ infra/gcp/main.tf | 114 +++++++++++++++++++++ infra/gcp/variables.tf | 27 +++++ 9 files changed, 315 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 dags/ledger_daily_pipeline.py create mode 100644 dbt/dbt_project.yml create mode 100644 dbt/models/marts/fct_financial_facts.sql create mode 100644 dbt/models/schema.yml create mode 100644 dbt/models/staging/stg_financial_facts.sql create mode 100644 dbt/profiles.yml create mode 100644 infra/gcp/main.tf create mode 100644 infra/gcp/variables.tf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f8bf944 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - run: uv sync --group dev + - run: uv run ruff check src tests + - run: uv run mypy src + - run: uv run pytest + - name: Fixture pipeline smoke test + run: | + cp config/ledger.example.yml config/ledger.yml + uv run python -m ledger.cli --config config/ledger.yml --fixture-dir tests/fixtures + - run: uv run dbt build --project-dir dbt --profiles-dir dbt diff --git a/dags/ledger_daily_pipeline.py b/dags/ledger_daily_pipeline.py new file mode 100644 index 0000000..cc49561 --- /dev/null +++ b/dags/ledger_daily_pipeline.py @@ -0,0 +1,37 @@ +"""Example Airflow DAG for the reusable Project Ledger template.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.bash import BashOperator + +PROJECT_DIR = os.environ.get("LEDGER_PROJECT_DIR", "/opt/project-ledger") +CONFIG_PATH = os.environ.get("LEDGER_CONFIG_PATH", "config/ledger.yml") + +with DAG( + dag_id="ledger_daily_pipeline", + start_date=datetime(2025, 1, 1), + schedule="0 6 * * *", + catchup=False, + default_args={"retries": 2, "retry_delay": timedelta(minutes=5)}, + tags=["ledger", "sec", "template"], +) as dag: + ingest = BashOperator( + task_id="ingest_companyfacts", + bash_command=( + f"cd {PROJECT_DIR} && uv run python -m ledger.cli " + f"--config {CONFIG_PATH} --live" + ), + ) + + build = BashOperator( + task_id="build_dbt_models", + bash_command=( + f"cd {PROJECT_DIR} && uv run dbt build --project-dir dbt --profiles-dir dbt" + ), + ) + + ingest >> build diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml new file mode 100644 index 0000000..b106ec5 --- /dev/null +++ b/dbt/dbt_project.yml @@ -0,0 +1,20 @@ +name: ledger +version: "1.0.0" +config-version: 2 +profile: ledger + +model-paths: ["models"] +clean-targets: ["target", "dbt_packages"] + +vars: + normalized_facts_path: "../data/normalized/financial_facts.jsonl" + +models: + ledger: + +persist_docs: + relation: true + columns: true + staging: + +materialized: view + marts: + +materialized: table diff --git a/dbt/models/marts/fct_financial_facts.sql b/dbt/models/marts/fct_financial_facts.sql new file mode 100644 index 0000000..88bbde7 --- /dev/null +++ b/dbt/models/marts/fct_financial_facts.sql @@ -0,0 +1,35 @@ +with facts as ( + select * from {{ ref('stg_financial_facts') }} +), + +deduplicated as ( + select + *, + row_number() over ( + partition by cik, fact_namespace, fact_name, unit, accession_number, period_end_date + order by filed_date desc + ) as row_number + from facts +) + +select + md5(concat_ws('|', cik, fact_namespace, fact_name, unit, accession_number, period_end_date)) + as financial_fact_id, + cik, + entity_name, + fact_namespace, + fact_name, + fact_label, + fact_description, + unit, + value, + accession_number, + fiscal_year, + fiscal_period, + filing_form, + filed_date, + frame, + period_start_date, + period_end_date +from deduplicated +where row_number = 1 diff --git a/dbt/models/schema.yml b/dbt/models/schema.yml new file mode 100644 index 0000000..77fa283 --- /dev/null +++ b/dbt/models/schema.yml @@ -0,0 +1,25 @@ +version: 2 + +models: + - name: stg_financial_facts + description: Normalized SEC XBRL observations prepared for analytical modeling. + columns: + - name: cik + description: SEC Central Index Key, left-padded to ten digits. + data_tests: [not_null] + - name: fact_name + description: XBRL fact identifier within its taxonomy namespace. + data_tests: [not_null] + - name: accession_number + description: SEC filing accession number associated with the observation. + data_tests: [not_null] + + - name: fct_financial_facts + description: Deduplicated analytical fact table containing one row per filing observation grain. + columns: + - name: financial_fact_id + description: Deterministic identifier for the analytical observation grain. + data_tests: [not_null, unique] + - name: cik + description: SEC Central Index Key for the reporting entity. + data_tests: [not_null] diff --git a/dbt/models/staging/stg_financial_facts.sql b/dbt/models/staging/stg_financial_facts.sql new file mode 100644 index 0000000..a01f707 --- /dev/null +++ b/dbt/models/staging/stg_financial_facts.sql @@ -0,0 +1,23 @@ +with source as ( + select * + from read_json_auto('{{ var("normalized_facts_path") }}', format = 'newline_delimited') +) + +select + cast(cik as varchar) as cik, + cast(entity_name as varchar) as entity_name, + cast(namespace as varchar) as fact_namespace, + cast(fact_name as varchar) as fact_name, + cast(label as varchar) as fact_label, + cast(description as varchar) as fact_description, + cast(unit as varchar) as unit, + value, + cast(accession_number as varchar) as accession_number, + cast(fiscal_year as integer) as fiscal_year, + cast(fiscal_period as varchar) as fiscal_period, + cast(form as varchar) as filing_form, + cast(filed_date as date) as filed_date, + cast(frame as varchar) as frame, + cast(period_start_date as date) as period_start_date, + cast(period_end_date as date) as period_end_date +from source diff --git a/dbt/profiles.yml b/dbt/profiles.yml new file mode 100644 index 0000000..4040874 --- /dev/null +++ b/dbt/profiles.yml @@ -0,0 +1,7 @@ +ledger: + target: dev + outputs: + dev: + type: duckdb + path: "../data/ledger.duckdb" + threads: 4 diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf new file mode 100644 index 0000000..c021b2a --- /dev/null +++ b/infra/gcp/main.tf @@ -0,0 +1,114 @@ +terraform { + required_version = ">= 1.7" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +locals { + name = "ledger-${var.environment}" +} + +resource "google_project_service" "services" { + for_each = toset([ + "bigquery.googleapis.com", + "run.googleapis.com", + "secretmanager.googleapis.com", + "storage.googleapis.com", + ]) + project = var.project_id + service = each.value + disable_on_destroy = false +} + +resource "google_service_account" "job" { + account_id = "${local.name}-job" + display_name = "Project Ledger ${var.environment} job" +} + +resource "google_storage_bucket" "data" { + name = "${var.project_id}-${local.name}-data" + location = var.region + uniform_bucket_level_access = true + force_destroy = false +} + +resource "google_bigquery_dataset" "raw" { + dataset_id = "ledger_raw_${var.environment}" + location = var.region +} + +resource "google_bigquery_dataset" "mart" { + dataset_id = "ledger_mart_${var.environment}" + location = var.region +} + +resource "google_project_iam_member" "bigquery_job_user" { + project = var.project_id + role = "roles/bigquery.jobUser" + member = "serviceAccount:${google_service_account.job.email}" +} + +resource "google_bigquery_dataset_iam_member" "raw_editor" { + dataset_id = google_bigquery_dataset.raw.dataset_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.job.email}" +} + +resource "google_bigquery_dataset_iam_member" "mart_editor" { + dataset_id = google_bigquery_dataset.mart.dataset_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.job.email}" +} + +resource "google_storage_bucket_iam_member" "object_admin" { + bucket = google_storage_bucket.data.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.job.email}" +} + +resource "google_cloud_run_v2_job" "ledger" { + name = local.name + location = var.region + + template { + template { + service_account = google_service_account.job.email + containers { + image = var.container_image + args = ["--config", "config/ledger.yml", "--live"] + env { + name = "LEDGER_OUTPUT_DIR" + value = "/tmp/data" + } + env { + name = "LEDGER_SEC_USER_AGENT" + value_source { + secret_key_ref { + secret = var.sec_user_agent_secret_id + version = "latest" + } + } + } + } + } + } + + depends_on = [google_project_service.services] +} + +output "cloud_run_job_name" { + value = google_cloud_run_v2_job.ledger.name +} + +output "data_bucket_name" { + value = google_storage_bucket.data.name +} diff --git a/infra/gcp/variables.tf b/infra/gcp/variables.tf new file mode 100644 index 0000000..1d66108 --- /dev/null +++ b/infra/gcp/variables.tf @@ -0,0 +1,27 @@ +variable "project_id" { + description = "GCP project that will host Project Ledger resources." + type = string +} + +variable "region" { + description = "GCP region for regional resources." + type = string + default = "us-central1" +} + +variable "environment" { + description = "Deployment environment name." + type = string + default = "dev" +} + +variable "container_image" { + description = "Fully qualified container image for the Cloud Run Job." + type = string +} + +variable "sec_user_agent_secret_id" { + description = "Secret Manager secret containing the SEC user-agent string." + type = string + default = "ledger-sec-user-agent" +} From 2b84e893bfbd657a869238b718b135be4697179c Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:42:33 -0500 Subject: [PATCH 04/15] fix static analysis and dbt path resolution --- dbt/dbt_project.yml | 2 +- dbt/profiles.yml | 2 +- src/ledger/config.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index b106ec5..a021210 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -7,7 +7,7 @@ model-paths: ["models"] clean-targets: ["target", "dbt_packages"] vars: - normalized_facts_path: "../data/normalized/financial_facts.jsonl" + normalized_facts_path: "data/normalized/financial_facts.jsonl" models: ledger: diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 4040874..946918d 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -3,5 +3,5 @@ ledger: outputs: dev: type: duckdb - path: "../data/ledger.duckdb" + path: "data/ledger.duckdb" threads: 4 diff --git a/src/ledger/config.py b/src/ledger/config.py index c27d88b..c103501 100644 --- a/src/ledger/config.py +++ b/src/ledger/config.py @@ -1,9 +1,10 @@ from __future__ import annotations import os +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any, Mapping +from typing import Any import yaml From 7ae13d29697dce51a08d88278516931c9b9c798f Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:45:16 -0500 Subject: [PATCH 05/15] capture Ruff diagnostics in CI --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8bf944..ec608ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,14 @@ jobs: with: python-version: "3.12" - run: uv sync --group dev - - run: uv run ruff check src tests + - name: Ruff + run: uv run ruff check src tests --output-format=full 2>&1 | tee ruff-output.txt + - name: Upload Ruff diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ruff-diagnostics + path: ruff-output.txt - run: uv run mypy src - run: uv run pytest - name: Fixture pipeline smoke test From c0bbefe0d43f6dd8b75c0502ae97430d5bc58200 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:45:58 -0500 Subject: [PATCH 06/15] capture mypy diagnostics in CI --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec608ec..40cc7f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,14 @@ jobs: with: name: ruff-diagnostics path: ruff-output.txt - - run: uv run mypy src + - name: Mypy + run: uv run mypy src 2>&1 | tee mypy-output.txt + - name: Upload mypy diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: mypy-diagnostics + path: mypy-output.txt - run: uv run pytest - name: Fixture pipeline smoke test run: | From 7bbf28bcde8391091f366e0db0930fa936b8bdfb Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:47:21 -0500 Subject: [PATCH 07/15] finalize CI and validate Terraform --- .github/workflows/ci.yml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40cc7f1..56673dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,24 +18,23 @@ jobs: python-version: "3.12" - run: uv sync --group dev - name: Ruff - run: uv run ruff check src tests --output-format=full 2>&1 | tee ruff-output.txt - - name: Upload Ruff diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: ruff-diagnostics - path: ruff-output.txt + run: uv run ruff check src tests - name: Mypy - run: uv run mypy src 2>&1 | tee mypy-output.txt - - name: Upload mypy diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: mypy-diagnostics - path: mypy-output.txt - - run: uv run pytest + run: uv run mypy src + - name: Pytest + run: uv run pytest - name: Fixture pipeline smoke test run: | cp config/ledger.example.yml config/ledger.yml uv run python -m ledger.cli --config config/ledger.yml --fixture-dir tests/fixtures - - run: uv run dbt build --project-dir dbt --profiles-dir dbt + - name: dbt build + run: uv run dbt build --project-dir dbt --profiles-dir dbt + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.8" + - name: Terraform format + run: terraform -chdir=infra/gcp fmt -check -recursive + - name: Terraform validate + run: | + terraform -chdir=infra/gcp init -backend=false + terraform -chdir=infra/gcp validate From fceca9797c1aca513055afe636a54e0f24ea8fc9 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:49:40 -0500 Subject: [PATCH 08/15] fix lint and strict typing findings --- pyproject.toml | 4 ++++ src/ledger/cli.py | 7 ++++++- src/ledger/sec_client.py | 13 ++++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f5692c8..b68139a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,10 @@ strict = true [tool.ruff] line-length = 100 target-version = "py312" +src = ["src"] [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] + +[tool.ruff.lint.isort] +known-first-party = ["ledger"] diff --git a/src/ledger/cli.py b/src/ledger/cli.py index faa4449..6bfa6ac 100644 --- a/src/ledger/cli.py +++ b/src/ledger/cli.py @@ -6,7 +6,11 @@ from ledger.config import load_config from ledger.pipeline import run_pipeline -from ledger.sec_client import FixtureCompanyFactsClient, SecCompanyFactsClient +from ledger.sec_client import ( + CompanyFactsClient, + FixtureCompanyFactsClient, + SecCompanyFactsClient, +) def build_parser() -> argparse.ArgumentParser: @@ -21,6 +25,7 @@ def build_parser() -> argparse.ArgumentParser: def main() -> int: args = build_parser().parse_args() config = load_config(Path(args.config)) + client: CompanyFactsClient if args.live: config.require_live_access() diff --git a/src/ledger/sec_client.py b/src/ledger/sec_client.py index 1b973f2..1d0879f 100644 --- a/src/ledger/sec_client.py +++ b/src/ledger/sec_client.py @@ -3,7 +3,7 @@ import json import time from pathlib import Path -from typing import Any, Protocol +from typing import Any, Protocol, cast from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -12,6 +12,13 @@ class CompanyFactsClient(Protocol): def fetch_companyfacts(self, cik: str) -> dict[str, Any]: ... +def _decode_payload(content: str, source: str) -> dict[str, Any]: + payload: object = json.loads(content) + if not isinstance(payload, dict): + raise ValueError(f"Expected a JSON object from {source}") + return cast(dict[str, Any], payload) + + class SecCompanyFactsClient: BASE_URL = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json" @@ -49,7 +56,7 @@ def fetch_companyfacts(self, cik: str) -> dict[str, Any]: try: self._last_request_at = time.monotonic() with urlopen(request, timeout=self.timeout_seconds) as response: # noqa: S310 - return json.loads(response.read().decode("utf-8")) + return _decode_payload(response.read().decode("utf-8"), url) except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error: last_error = error if attempt >= self.max_retries: @@ -67,4 +74,4 @@ def fetch_companyfacts(self, cik: str) -> dict[str, Any]: fixture_path = self.fixture_dir / f"CIK{cik.zfill(10)}.json" if not fixture_path.exists(): raise FileNotFoundError(f"Missing fixture: {fixture_path}") - return json.loads(fixture_path.read_text(encoding="utf-8")) + return _decode_payload(fixture_path.read_text(encoding="utf-8"), str(fixture_path)) From b36b9b596b9ef47ab39716f46ddb85b54843a34a Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:51:16 -0500 Subject: [PATCH 09/15] preserve Ruff diagnostics without masking failures --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56673dd..3a2d0fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,16 @@ jobs: python-version: "3.12" - run: uv sync --group dev - name: Ruff - run: uv run ruff check src tests + run: | + set -o pipefail + uv run ruff check src tests --output-format=full 2>&1 | tee ruff-output.txt + - name: Upload Ruff diagnostics + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ruff-diagnostics + path: ruff-output.txt + retention-days: 7 - name: Mypy run: uv run mypy src - name: Pytest From 1b3bd792ea9dc598096b6281651bfac90c1cd520 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:52:36 -0500 Subject: [PATCH 10/15] fix test import spacing --- tests/test_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index aa37c7c..ec1b047 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,7 +4,6 @@ from ledger.config import ConfigurationError, load_config - CONFIG = """ project: name: template-test From 853f3c3db12e165a4a479347f23f4864373cefd6 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:52:49 -0500 Subject: [PATCH 11/15] fix pipeline test import spacing --- tests/test_pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 9b30e99..cfc7eb4 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,7 +5,6 @@ from ledger.pipeline import normalize_companyfacts, run_pipeline from ledger.sec_client import FixtureCompanyFactsClient - FIXTURE_DIR = Path(__file__).parent / "fixtures" From fb1fef43679a872c79d14be3f3a5000847674fa4 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:18:57 -0500 Subject: [PATCH 12/15] docs: add Project Ledger case study --- docs/CASE_STUDY.md | 118 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 docs/CASE_STUDY.md diff --git a/docs/CASE_STUDY.md b/docs/CASE_STUDY.md new file mode 100644 index 0000000..7348dc9 --- /dev/null +++ b/docs/CASE_STUDY.md @@ -0,0 +1,118 @@ +# Project Ledger Case Study + +## Context + +Project Ledger began as a production-oriented learning implementation for ingesting, preserving, transforming, and serving public SEC/XBRL financial data. The implementation was later generalized into this organization-neutral reference repository so another engineer can study, run, and extend the architecture without inheriting personal infrastructure, credentials, or project-specific history. + +This repository is therefore not a byte-for-byte export of an operating environment. It is the reusable engineering core extracted from a broader system. + +## Engineering objective + +Build a financial-data pipeline that could move from deterministic local development to managed cloud execution without changing its core data contracts. + +The system needed to: + +- support deterministic development without depending on a live API +- preserve source payloads before transformation +- normalize semi-structured XBRL observations into a stable analytical grain +- prevent duplicate analytical records across repeated runs +- separate configuration from code and identity +- validate code, models, infrastructure, and pipeline behavior in CI +- expose clear extension points for orchestration, storage, governance, and observability + +## Implemented scope + +The reusable template includes: + +- Python-based SEC Company Facts ingestion +- live and fixture-backed clients behind a common protocol +- governed issuer configuration and CIK normalization +- raw JSON preservation +- normalized JSONL financial observations +- atomic and repeatable local writes +- dbt staging and deduplicated fact models in DuckDB +- unit and pipeline tests +- strict type checking and linting +- Docker packaging +- an Airflow DAG example +- parameterized GCP infrastructure for Cloud Run Jobs, GCS, BigQuery, IAM, and Secret Manager references +- GitHub Actions validation across application, analytics, and infrastructure layers + +The broader implementation from which this template was extracted also explored managed GCP execution, BigQuery serving, Airflow or Composer orchestration, workload identity, monitoring, market-price enrichment, machine-learning extensions, language-model-assisted analysis, and earnings-event processing. Those later extensions are described as future integration patterns rather than presented here as completed template features. + +## Key engineering problems + +### 1. Reliable development against an external API + +Depending entirely on live SEC requests would make tests slower, less deterministic, and more vulnerable to rate limits or network failures. The solution was a fixture-first client boundary. The same pipeline can consume committed payloads in CI or a rate-limited live client in an authorized runtime. + +### 2. Preserving source truth while supporting analytics + +Transforming source payloads in place would remove evidence needed for debugging and future reprocessing. The pipeline therefore writes the original issuer payload separately from normalized observations. + +### 3. Repeated execution without duplicate facts + +Financial facts may contain repeated observations across forms and filing periods. The pipeline produces governed outputs atomically, while dbt applies a documented analytical grain and deduplication rule before exposing the mart. + +### 4. Avoiding environmental coupling + +Personal names, cloud identifiers, issuer selections, paths, and runtime contacts were moved into configuration or Terraform variables. This allows the implementation to be reused without editing application logic. + +### 5. Validating the entire delivery surface + +A passing unit-test suite alone would not establish readiness. CI validates Python formatting and typing, tests, a fixture pipeline smoke run, dbt models and data tests, and Terraform formatting and syntax. + +## Failure modes considered + +The design explicitly accounts for: + +- invalid or mismatched issuer identifiers +- malformed Company Facts payloads +- API rate limits and transient request failures +- missing environment variables +- partial writes +- duplicate analytical observations +- model contract drift +- unsafe hard-coded infrastructure values +- production dependencies leaking into deterministic CI + +## What the project demonstrates + +This project provides evidence of competence in: + +- ingestion boundary design +- semi-structured data normalization +- analytical data modeling +- idempotency and reproducibility +- configuration and secrets separation +- automated testing and delivery gates +- orchestration design +- containerization +- infrastructure as code +- converting a project-specific system into a reusable engineering template + +## Limitations + +The repository is a reference implementation, not a hosted production service. It does not include: + +- a live deployed environment +- service-level objectives or production alert routing +- backfill coordination across multiple workers +- a complete BigQuery loading implementation +- full schema-evolution automation +- cost benchmarks at production volume +- generated dashboards, runtime logs, or private operational evidence + +These exclusions are deliberate. They prevent the public template from implying deployment evidence it does not contain. + +## Next steps + +The most valuable extensions would be: + +1. implement storage adapters for GCS and BigQuery +2. add explicit schema contracts and migration checks +3. add freshness, volume, error-rate, and cost observability +4. add integration tests against an ephemeral cloud environment +5. add backfill and replay controls +6. publish synthetic-data dashboards over the governed mart +7. document benchmark results and operational recovery exercises From f169aaf35fc2cab7921b5d119481f2559be7d7df Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:19:34 -0500 Subject: [PATCH 13/15] docs: add engineering decision record --- docs/DECISIONS.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docs/DECISIONS.md diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md new file mode 100644 index 0000000..ffdca7f --- /dev/null +++ b/docs/DECISIONS.md @@ -0,0 +1,83 @@ +# Engineering Decisions + +This document records the major architectural choices in the reusable Project Ledger template. It is intentionally concise. Production teams should convert decisions that materially affect their environment into formal ADRs with dates, owners, alternatives, and review status. + +## Decision 1: Use fixture-first development + +**Decision:** The ingestion pipeline must run from committed SEC Company Facts fixtures as well as the live API. + +**Why:** Deterministic fixtures make local development and CI independent of network access, API availability, and rate limits. + +**Trade-off:** Fixtures can drift from current source behavior. The project should periodically refresh them and retain a separate live-integration validation lane. + +## Decision 2: Keep live and fixture clients behind one protocol + +**Decision:** Pipeline code depends on a `CompanyFactsClient` boundary rather than a concrete HTTP implementation. + +**Why:** The ingestion and normalization path can be exercised without conditional logic scattered throughout the pipeline. + +**Trade-off:** The abstraction must remain narrow. Adding source-specific behavior to the protocol would reduce substitutability. + +## Decision 3: Preserve raw payloads before normalization + +**Decision:** Store the complete source payload separately from normalized analytical observations. + +**Why:** Raw preservation supports replay, debugging, auditability, and future transformation changes. + +**Trade-off:** Raw storage increases retention and governance requirements. Production deployments must define lifecycle, classification, and access policies. + +## Decision 4: Define issuers through an allowlist + +**Decision:** Process only explicitly configured issuer names and CIKs. + +**Why:** An allowlist makes scope intentional, reduces unexpected volume, and creates a validation boundary between configuration and source payloads. + +**Trade-off:** Expanding coverage requires a configuration change. Large-scale discovery would need a governed issuer registry rather than a static file. + +## Decision 5: Normalize to an observation-level analytical grain + +**Decision:** Emit individual financial observations with issuer, taxonomy, concept, unit, period, filing, form, and value attributes. + +**Why:** A narrow fact grain supports flexible downstream modeling while preserving the filing context required for deduplication. + +**Trade-off:** XBRL semantics are complex. Production marts may need concept mapping, dimensional context, restatement rules, and company-specific exceptions. + +## Decision 6: Make repeated runs replace governed local outputs + +**Decision:** Write raw and normalized files atomically and replace the governed output for the same configured run scope. + +**Why:** This provides simple repeatability and avoids accidental append-only duplication in the reference implementation. + +**Trade-off:** Replacement is not a complete production backfill strategy. Cloud implementations should use partition-aware loads, merge semantics, checkpoints, and immutable raw landing paths. + +## Decision 7: Separate application, analytics, and infrastructure validation + +**Decision:** CI validates Python, the fixture pipeline, dbt, and Terraform in one delivery gate. + +**Why:** A data platform can fail outside application unit tests. Cross-layer validation exposes contract and packaging errors earlier. + +**Trade-off:** Validation time and dependency count increase. Production repositories may split fast pull-request checks from slower integration workflows. + +## Decision 8: Use DuckDB locally and keep cloud storage replaceable + +**Decision:** Use DuckDB for the runnable local analytical path while documenting GCS and BigQuery as production adapters. + +**Why:** DuckDB minimizes setup cost and makes the complete template executable by another engineer. + +**Trade-off:** Local execution does not prove distributed scale, cloud IAM, or BigQuery cost behavior. Those require separate deployment evidence. + +## Decision 9: Parameterize infrastructure and keep secrets external + +**Decision:** Cloud names and identifiers are Terraform variables; runtime contact information is supplied through environment variables or a secret reference. + +**Why:** Code should not encode a person's identity, a private environment, or credentials. + +**Trade-off:** Deployments require an explicit configuration process and external secret provisioning. + +## Decision 10: Generalize the implementation rather than publish operational history + +**Decision:** Exclude personal sprint journals, private identifiers, generated operational evidence, and environment-specific artifacts from the reusable repository. + +**Why:** The public artifact should demonstrate transferable engineering without leaking context or overstating what is deployed in the template. + +**Trade-off:** The repository alone provides less evidence of long-running operation. The case study documents scope and limitations without exposing private material. From 08a753b436f3d5a6cd2a7ea57a1c7e255e33d896 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:20:27 -0500 Subject: [PATCH 14/15] docs: add operational runbook --- docs/OPERATIONS.md | 185 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 docs/OPERATIONS.md diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..c9f01ff --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,185 @@ +# Operations Guide + +This guide describes the operational expectations for running the Project Ledger template. The repository ships with a deterministic local path and a cloud deployment skeleton. Teams must adapt the controls below before treating it as a production service. + +## Operating modes + +### Fixture mode + +Use fixture mode for local development, CI, debugging, and transformation validation. + +```bash +uv run python -m ledger.cli \ + --config config/ledger.yml \ + --fixture-dir tests/fixtures +``` + +Expected behavior: + +- no external SEC request is made +- configured issuer identities are validated against fixture payloads +- raw JSON and normalized JSONL outputs are written atomically +- repeated runs produce the same governed outputs + +### Live mode + +Use live mode only after setting a descriptive, monitored SEC user agent. + +```bash +export LEDGER_SEC_USER_AGENT="YourCompany Data Engineering your-team@example.com" +uv run python -m ledger.cli --config config/ledger.yml --live +``` + +Before enabling scheduled live execution: + +- confirm the issuer allowlist +- verify the runtime contact is monitored +- define request-rate and retry policies +- define raw-data retention +- configure secret storage +- verify destination permissions +- establish alert ownership + +## Routine validation + +Run the full local validation suite before deployment: + +```bash +bash scripts/validate.sh +``` + +The validation path should confirm: + +- Ruff formatting and linting +- strict mypy checks +- Python tests +- fixture pipeline execution +- dbt build and data tests +- Terraform formatting and validation + +## Expected outputs + +The default local implementation produces: + +- one preserved raw Company Facts payload per configured issuer +- normalized financial observations in JSONL +- a DuckDB staging model +- a deduplicated financial-facts mart + +Validate output completeness against the configured issuer count and expected fixture set. Do not rely only on process exit status. + +## Recommended production monitoring + +At minimum, collect and alert on: + +| Signal | Purpose | +| --- | --- | +| pipeline success and duration | detect failed or degraded runs | +| last successful run timestamp | enforce freshness expectations | +| issuers requested versus completed | detect partial ingestion | +| source observations versus normalized rows | detect parsing or schema changes | +| rejected or malformed observations | expose data-quality failures | +| duplicate rate before mart deduplication | detect source or grain changes | +| raw and modeled storage growth | manage retention and cost | +| API response status and retry count | detect source instability or throttling | +| warehouse query and load cost | detect inefficient changes | + +Production thresholds should be based on observed baselines rather than arbitrary values. + +## Failure handling + +### Missing SEC user agent + +**Symptom:** Live execution stops before requesting data. + +**Response:** Set `LEDGER_SEC_USER_AGENT` through the approved secret or runtime configuration mechanism. Do not hard-code it. + +### Issuer identity mismatch + +**Symptom:** The returned payload does not match the configured CIK. + +**Response:** Stop processing that issuer. Confirm CIK normalization and the configuration source before retrying. Do not silently relabel the payload. + +### SEC rate limit or transient error + +**Symptom:** HTTP errors, throttling, or exhausted retries. + +**Response:** Preserve the failure context, avoid aggressive immediate retries, and rerun only the failed issuer set after the cooldown policy permits. + +### Malformed payload or source schema change + +**Symptom:** Payload decoding or normalization fails. + +**Response:** Preserve the raw response when safe, quarantine it from modeled outputs, compare it with the client contract, add a regression fixture, and update parsing only after defining the new expected shape. + +### Partial or interrupted write + +**Symptom:** Missing output or a temporary file remains. + +**Response:** Confirm that the governed target was not partially replaced. Remove abandoned temporary files after investigation, then rerun the same scope. Atomic replacement should protect the prior valid target. + +### dbt test failure + +**Symptom:** The staging or mart build completes with failed data tests. + +**Response:** Do not promote the output. Identify whether the failure is caused by source drift, normalization, grain definition, or deduplication. Add a fixture reproducing the issue before changing the model. + +### Infrastructure validation failure + +**Symptom:** Terraform formatting, initialization, or validation fails. + +**Response:** Correct the configuration before planning or applying. Validation success does not replace policy review, IAM review, cost review, or an environment-specific plan inspection. + +## Replay and backfill + +The local template replaces outputs for a governed run scope. A production implementation should add: + +- immutable raw landing paths organized by ingestion date and issuer +- checkpoints or run manifests +- partition-aware warehouse loading +- idempotent merge keys +- bounded replay parameters +- concurrency controls +- backfill-specific cost and rate limits + +Do not run an unrestricted historical backfill through the same schedule as routine incremental ingestion. + +## Security and access + +- Use workload identity instead of service-account keys. +- Store runtime configuration and contacts in Secret Manager or an equivalent service. +- Grant the execution identity only the storage, warehouse, logging, and secret permissions it needs. +- Separate deployment permissions from runtime permissions. +- Do not log secrets, complete authorization headers, or sensitive environment variables. +- Review raw-source retention and access before adding filing documents or derived text. + +## Deployment checklist + +Before the first managed run: + +- [ ] issuer configuration reviewed +- [ ] SEC user agent stored externally +- [ ] destination resources created in the intended project +- [ ] runtime identity uses least privilege +- [ ] CI identity is separate from runtime identity +- [ ] retention and deletion policies defined +- [ ] schema and dbt tests pass +- [ ] monitoring and alert ownership configured +- [ ] dry run or fixture-backed deployment completed +- [ ] rollback and replay procedures reviewed + +## Incident record + +For any material failure, capture: + +- run identifier and time window +- affected issuers and destinations +- first observed symptom +- source and modeled row counts +- relevant error class and retry behavior +- containment action +- root cause +- repair and replay scope +- test or control added to prevent recurrence + +Avoid storing credentials, private payloads, or unredacted production data in incident records committed to the repository. From 04bcdd1b7386ac05f96f0fefbca7106f8f2de0b9 Mon Sep 17 00:00:00 2001 From: Russell <109922707+rlancaster243@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:21:07 -0500 Subject: [PATCH 15/15] docs: reposition README as portfolio entry point --- README.md | 129 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 104 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 8a7a5f8..4c2ad77 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,23 @@ # Project Ledger -Project Ledger is a reusable reference implementation for building a production-oriented SEC/XBRL data pipeline. It demonstrates fixture-first development, governed issuer selection, idempotent ingestion, normalized financial facts, local DuckDB/dbt modeling, optional Airflow orchestration, and a parameterized GCP deployment path. +Project Ledger is a reusable reference implementation for building a production-oriented SEC/XBRL data platform. It was extracted from a broader engineering implementation and generalized so another engineer can run, study, and extend the architecture without inheriting private infrastructure, personal identifiers, credentials, or project-specific history. -The repository is intentionally organization-neutral. It contains no personal identifiers, private infrastructure names, fixed cloud project IDs, or embedded credentials. +The repository demonstrates how to move from deterministic local development toward managed cloud execution while preserving stable ingestion, storage, modeling, testing, and delivery boundaries. + +## What this project demonstrates + +- fixture-first API integration and deterministic CI +- governed issuer selection and CIK validation +- raw-source preservation before transformation +- normalization of semi-structured XBRL observations +- repeatable and idempotent pipeline behavior +- dbt staging, analytical grain definition, and deduplication +- strict typing, linting, unit tests, data tests, and smoke tests +- Airflow orchestration boundaries +- Docker packaging +- parameterized GCP infrastructure as code +- separation of code, configuration, identity, and secrets +- extraction of a project-specific implementation into a reusable engineering template ## Architecture @@ -25,7 +40,7 @@ raw JSON per issuer normalized JSONL facts analytical marts ``` -Optional production path: +Optional managed path: ```text GitHub Actions -> container image -> Cloud Run Job -> GCS/BigQuery @@ -34,23 +49,57 @@ GitHub Actions -> container image -> Cloud Run Job -> GCS/BigQuery Airflow/Composer ``` +The local implementation is intentionally runnable with DuckDB and committed fixtures. The cloud layer is parameterized so storage, execution, IAM, and orchestration can be replaced without rewriting the core ingestion contract. + +See [Architecture](docs/ARCHITECTURE.md) for component responsibilities and extension points. + +## Engineering scope + +The reusable template includes: + +- Python SEC Company Facts ingestion +- live and fixture-backed clients +- raw JSON and normalized JSONL outputs +- atomic local writes +- dbt/DuckDB staging and mart models +- Python and dbt tests +- GitHub Actions validation +- Docker packaging +- an Airflow DAG example +- Terraform for a GCP deployment skeleton + +The broader implementation from which this repository was extracted also explored managed GCP execution, BigQuery serving, Composer or Airflow orchestration, workload identity, monitoring, market-price enrichment, machine-learning extensions, language-model-assisted analysis, and earnings-event processing. Those later extensions are documented as integration directions rather than represented here as completed template features. + +## Key design decisions + +1. **Determinism before live integration.** The complete transformation path runs from committed fixtures. +2. **Raw preservation before normalization.** Original payloads remain available for replay and debugging. +3. **Configuration over identity.** Issuers, paths, cloud resources, and runtime contacts are externalized. +4. **Stable local-to-cloud contracts.** Core ingestion and modeling boundaries remain consistent as infrastructure changes. +5. **Validation across layers.** CI checks application code, pipeline behavior, analytical models, and infrastructure syntax. + +See [Engineering Decisions](docs/DECISIONS.md) for the rationale and trade-offs behind these choices. + ## Quick start -1. Install Python 3.12 and `uv`. -2. Copy the examples: +### 1. Install prerequisites + +Install Python 3.12 and `uv`. + +### 2. Create local configuration ```bash cp .env.example .env cp config/ledger.example.yml config/ledger.yml ``` -3. Set a descriptive SEC user agent in `.env`: +Set a descriptive SEC user agent in `.env`: ```text LEDGER_SEC_USER_AGENT=YourCompany Data Engineering your-team@example.com ``` -4. Install and validate: +### 3. Install and validate ```bash uv sync --group dev @@ -58,7 +107,7 @@ uv run pytest bash scripts/validate.sh ``` -5. Run from deterministic fixtures: +### 4. Run from deterministic fixtures ```bash uv run python -m ledger.cli \ @@ -66,34 +115,64 @@ uv run python -m ledger.cli \ --fixture-dir tests/fixtures ``` -6. Run against the live SEC API only after configuring a valid user agent: +### 5. Run against the live SEC API + +Use live mode only after configuring a valid, monitored user agent: ```bash uv run python -m ledger.cli --config config/ledger.yml --live ``` -7. Build the local warehouse: +### 6. Build the local warehouse ```bash uv run dbt build --project-dir dbt --profiles-dir dbt ``` +## Validation + +The delivery gate verifies: + +- Ruff formatting and linting +- strict mypy checks +- Python unit and pipeline tests +- a deterministic fixture ingestion smoke test +- dbt model builds and data tests +- Terraform formatting and validation + +A passing CI run establishes that the reference implementation is internally consistent. It does not by itself prove production scale, cloud deployment, or long-running operational reliability. + +## Documentation + +- [Case Study](docs/CASE_STUDY.md): origin, engineering objective, implemented scope, failure modes, evidence, limitations, and next steps +- [Architecture](docs/ARCHITECTURE.md): design goals, components, and production extension points +- [Engineering Decisions](docs/DECISIONS.md): major choices, rationale, and trade-offs +- [Operations Guide](docs/OPERATIONS.md): operating modes, monitoring, failure handling, replay, security, and deployment checks +- [Customization Guide](docs/CUSTOMIZATION.md): adapting the template to another organization or source system + ## Template boundaries -Included: -- Python ingestion and normalization -- fixture-based tests -- dbt/DuckDB models -- CI validation -- Airflow DAG example -- Docker packaging -- parameterized GCP Terraform skeleton -- architecture and customization guidance +This repository deliberately excludes: + +- personal sprint journals and career-development notes +- generated dashboards, runtime logs, and private operational evidence +- real cloud project IDs, bucket names, service accounts, personal emails, and secrets +- issuer selections tied to a specific person or organization +- claims that the template itself is a currently hosted production service + +These exclusions make the repository safe to reuse and keep its claims precise. The project should be evaluated as a production-oriented reference architecture and portfolio case study, not as proof of an active public deployment. + +## Limitations and next steps + +The current template does not include a complete managed warehouse loader, production alert routing, schema-migration automation, distributed backfill controls, or production cost benchmarks. + +The highest-value extensions are: -Deliberately excluded: -- personal sprint journals and validation history -- generated dashboard exports and runtime evidence -- real cloud project IDs, bucket names, service accounts, emails, and secrets -- issuer selections tied to a specific individual or organization +1. implement GCS and BigQuery storage adapters +2. add explicit schema contracts and migration checks +3. add freshness, volume, error-rate, and cost observability +4. add ephemeral cloud integration tests +5. add bounded replay and backfill controls +6. publish synthetic-data dashboards over the governed mart -See `docs/CUSTOMIZATION.md` before deploying. +See [Customizing the template](docs/CUSTOMIZATION.md) before deploying it in a real environment.