diff --git a/.gitignore b/.gitignore index a78e168c..bda18a70 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,7 @@ outputs/project_status_top_actions.csv outputs/project_status_next_steps.csv local_artifacts_backup/ .worktrees/ +/output/documents/Stock_Research_Command_Center_Interview_Brief.docx +/output/pdf/Stock_Research_Command_Center_Interview_Brief.pdf +/output/source/Stock_Research_Command_Center_Interview_Brief.py +/tmp/interview_brief/ diff --git a/Makefile b/Makefile index 6b5c8bdb..6eaa9c50 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help help-full next-stage profile-context research-change-snapshot research-change-monitor research-review-queue research-event-review-record demo demo-data-build demo-data-check demo-dashboard demo-dashboard-smoke demo-dashboard-render-smoke local-profile-seed linkedin-share-check public-performance-contract public-performance-gate public-ux-review-checklist public-ux-review-checklist-json public-ux-review-notes public-ux-review-notes-check public-ux-review-note pilot-review-feedback pilot-feedback-closeout browser-qa-evidence browser-qa-capture-plan hosted-demo-readiness pilot-readiness-check pilot-readiness-packet pilot-share-brief trusted-data-pilot trusted-data-pilot-candidates trusted-data-pilot-packet trusted-data-pilot-lane trusted-data-pilot-board trusted-data-pilot-evidence reviewed-data-proof reviewed-data-proof-record reviewed-batch-proof reviewed-batch-proof-record reviewed-batch-compare reviewed-batch-preflight proof-readiness-reconciliation auto-refresh-plan auto-refresh-daily auto-refresh-weekly auto-refresh-optional auto-refresh-runbook auto-refresh-status scheduler-activation-checklist auto-apply-gate lane-outcome-history price-reviewed-run fundamentals-batch-proof peer-batch-proof peer-mapping-source-review peer-mapping-writeback-guard public-demo-readiness-pack readiness-ops-center readiness-queue data-coverage-proof-queues coverage-frontier data-coverage-planner coverage-expansion-loop readiness-ops-evidence reviewed-batch decision-proof-queue metric-readiness metric-readiness-board benchmark-risk-review diff-hygiene diff-hygiene-summary diff-hygiene-files data-release-decision public-release-package public-release-handoff license-status session-source-preflight source-activation-guide provider-setup-checklist provider-smoke fmp-smoke alpha-vantage-smoke finnhub-smoke universe-scope fundamentals-source-ladder fundamentals-source-ladder-queue optional-context-source-ladder optional-context-source-ladder-queue fmp-stage alpha-vantage-stage finnhub-stage yfinance-stage sec-filing-share-stage staged-hygiene-check public-wording-check public-check status status-check test pipeline stock-report stock-report-md local-tickers monthly track-record validate-data data-sources-check data-sources research-health research-health-check risk-context action-queue action-queue-check project-status project-status-check verify validate-all daily dashboard dashboard-smoke sec-stage sec-validate sec-preview sec-apply imports-validate imports-preview imports-apply import-staging universe-preview universe-preview-summary universe-stage universe-apply universe-refresh universe-report universe-active coverage data-wizard unlock-ladder unlock-summary command-bundles command-bundle-details command-bundle-runbook bundle-prices bundle-fundamentals bundle-peers bundle-prices-broader bundle-fundamentals-broader bundle-peers-broader detail-prices detail-fundamentals detail-peers detail-prices-broader detail-fundamentals-broader detail-peers-broader runbook-prices runbook-fundamentals runbook-peers runbook-prices-broader runbook-fundamentals-broader runbook-peers-broader focus-price focus-fundamentals focus-peers onboarding templates price-status price-worklist fundamentals-peer-worklist optional-context-worklist sec-stage-queue peer-mapping-queue dcf-input-proof-queue dcf-input-proof-handoff dcf-input-source-review dcf-input-source-command-plan dcf-input-source-guard share-count-proof-queue price-history-proof-queue price-validate price-preview price-apply price-refresh price-refresh-loop price-normalize import-prices price-coverage dcf-readiness import-fundamentals optional-context-summary optional-context-readiness import-earnings import-analyst-estimates readiness readiness-preview readiness-release-review readiness-release-record readiness-release-guard readiness-materialize readiness-snapshot research-decisions earnings-nowcast-pilot earnings-nowcast-walkthrough earnings-nowcast-templates earnings-nowcast-validate earnings-nowcast-preview earnings-nowcast-readiness earnings-nowcast-prospective-plan +.PHONY: help help-full next-stage profile-context research-change-snapshot research-change-monitor research-review-queue research-event-review-record demo demo-data-build demo-data-check demo-dashboard demo-dashboard-smoke demo-dashboard-render-smoke local-profile-seed linkedin-share-check public-performance-contract public-performance-gate public-ux-review-checklist public-ux-review-checklist-json public-ux-review-notes public-ux-review-notes-check public-ux-review-note pilot-review-feedback pilot-feedback-closeout browser-qa-evidence browser-qa-capture-plan hosted-demo-readiness pilot-readiness-check pilot-readiness-packet pilot-share-brief trusted-data-pilot trusted-data-pilot-candidates trusted-data-pilot-packet trusted-data-pilot-lane trusted-data-pilot-board trusted-data-pilot-evidence reviewed-data-proof reviewed-data-proof-record reviewed-batch-proof reviewed-batch-proof-record reviewed-batch-compare reviewed-batch-preflight proof-readiness-reconciliation auto-refresh-plan auto-refresh-daily auto-refresh-weekly auto-refresh-optional auto-refresh-runbook auto-refresh-status scheduler-activation-checklist auto-apply-gate lane-outcome-history price-reviewed-run fundamentals-batch-proof peer-batch-proof peer-mapping-source-review peer-mapping-writeback-guard public-demo-readiness-pack readiness-ops-center readiness-queue data-coverage-proof-queues coverage-frontier data-coverage-planner coverage-expansion-loop readiness-ops-evidence reviewed-batch decision-proof-queue metric-readiness metric-readiness-board benchmark-risk-review diff-hygiene diff-hygiene-summary diff-hygiene-files data-release-decision public-release-package public-release-handoff license-status session-source-preflight source-activation-guide provider-setup-checklist provider-smoke fmp-smoke alpha-vantage-smoke finnhub-smoke universe-scope fundamentals-source-ladder fundamentals-source-ladder-queue optional-context-source-ladder optional-context-source-ladder-queue fmp-stage alpha-vantage-stage finnhub-stage yfinance-stage sec-filing-share-stage staged-hygiene-check public-wording-check public-check status status-check test pipeline stock-report stock-report-md local-tickers monthly track-record validate-data data-sources-check data-sources research-health research-health-check risk-context action-queue action-queue-check project-status project-status-check verify validate-all daily dashboard dashboard-smoke sec-stage sec-validate sec-preview sec-apply imports-validate imports-preview imports-apply import-staging universe-preview universe-preview-summary universe-stage universe-apply universe-refresh universe-report universe-active coverage data-wizard unlock-ladder unlock-summary command-bundles command-bundle-details command-bundle-runbook bundle-prices bundle-fundamentals bundle-peers bundle-prices-broader bundle-fundamentals-broader bundle-peers-broader detail-prices detail-fundamentals detail-peers detail-prices-broader detail-fundamentals-broader detail-peers-broader runbook-prices runbook-fundamentals runbook-peers runbook-prices-broader runbook-fundamentals-broader runbook-peers-broader focus-price focus-fundamentals focus-peers onboarding templates price-status price-worklist fundamentals-peer-worklist optional-context-worklist sec-stage-queue peer-mapping-queue dcf-input-proof-queue dcf-input-proof-handoff dcf-input-source-review dcf-input-source-command-plan dcf-input-source-guard share-count-proof-queue price-history-proof-queue price-validate price-preview price-apply price-refresh price-refresh-loop price-normalize import-prices price-coverage dcf-readiness import-fundamentals optional-context-summary optional-context-readiness import-earnings import-analyst-estimates readiness readiness-preview readiness-evidence-remediation readiness-release-review readiness-release-record readiness-release-guard readiness-materialize readiness-snapshot research-decisions earnings-nowcast-pilot earnings-nowcast-walkthrough earnings-nowcast-templates earnings-nowcast-validate earnings-nowcast-preview earnings-nowcast-readiness earnings-nowcast-prospective-plan .PHONY: thesis-journal thesis-journal-preview thesis-journal-record .PHONY: prospective-field-proof-status prospective-field-proof-audit prospective-field-proof-preview prospective-field-proof-record @@ -331,6 +331,7 @@ help-full: @echo "" @echo "Advanced readiness boundaries:" @echo " make readiness-preview [TOP_N=20] In-memory preview: stable readiness impact, change causes, and promotion evidence without writing files" + @echo " make readiness-evidence-remediation [TOP_N=20] [JSON=1] Closest-first inspection-only evidence queue; stdout only" @echo " make readiness-release-review [TOP_N=20] [JSON=1] Review the exact default readiness candidate without writing" @echo " make readiness-release-record PREVIEW_RECEIPT= REVIEWER= REVIEW_DATE= TECHNICAL_DECISION= DISTRIBUTION_DECISION= CONFIRM_REVIEWED=1 Record one exact named review" @echo " make readiness-release-guard RECORD_ID= Revalidate one record and print exact named staging paths" @@ -1482,6 +1483,9 @@ readiness: readiness-preview: @PYTHONDONTWRITEBYTECODE=1 python3 -m src.readiness_preview --top-n $(or $(TOP_N),20) +readiness-evidence-remediation: + @PYTHONDONTWRITEBYTECODE=1 python3 -m src.readiness_evidence_remediation --top-n $(or $(TOP_N),20) $(if $(JSON),--json,) + readiness-release-review: @PYTHONDONTWRITEBYTECODE=1 python3 -m src.readiness_release_review review --project-root . --top-n $(or $(TOP_N),20) $(if $(JSON),--json,) diff --git a/docs/OPERATOR_GUIDE.md b/docs/OPERATOR_GUIDE.md index b1e09b77..75ea7338 100644 --- a/docs/OPERATOR_GUIDE.md +++ b/docs/OPERATOR_GUIDE.md @@ -76,9 +76,12 @@ Inspect readiness impact after changing reviewed source data or imports; do not ```bash make pipeline # no-write guarded composite verification make readiness-preview TOP_N=20 +make readiness-evidence-remediation TOP_N=20 make project-status-check ``` +When the preview contains many technical promotions, `make readiness-evidence-remediation TOP_N=20` ranks the closest evidence-review candidates while keeping provenance, exact-source rights, registered field scope, price lineage, and timing blockers independent. Add `JSON=1` for deterministic structured stdout. The queue writes nothing and cannot apply canonical rows, change source rights, materialize readiness, or promote a ticker. + An optional ignored local package requires explicit authorization through `CONFIRM_MATERIALIZE=1 make readiness-materialize PROFILE=` and is not tracked release evidence. `make readiness-release-review TOP_N=20` applies only when an exact existing tracked 18-file candidate has been separately identified for review. `make project-status` also writes `outputs/project_status_remaining_stages.csv`. Use that stage map when deciding what is truly next after the public package is ready: LinkedIn manual share, hosted Streamlit deployment, FMP provider activation, peer readiness upgrade, optional earnings/estimate context, source-proof queue exhaustion, coverage-depth gaps, UX polish, and generated-artifact hygiene. The stage map is read-only; it separates actionable pending dependencies (`awaiting_external_setup`, `awaiting_reviewed_source`, and `awaiting_source_change`) from ready, manual-verification, and excluded states. These pending states do not turn the overall roadmap terminal: preserve their evidence, continue another executable product/share item, and resume only when the dependency changes. Lower-level tools may still report precise diagnostics such as `external_account_required` or `external_key_required`. diff --git a/src/readiness_evidence_remediation.py b/src/readiness_evidence_remediation.py new file mode 100644 index 00000000..fe271159 --- /dev/null +++ b/src/readiness_evidence_remediation.py @@ -0,0 +1,375 @@ +"""Deterministic, stdout-only remediation queue for proposed readiness evidence.""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Mapping + +from src.commercial_source_rights import SourceRights +from src.dcf_price_lineage import DcfPriceLineageEvidence +from src.readiness_preview import ( + ReadinessImpactPreview, + ReadinessPromotionEvidence, + build_readiness_impact_preview, +) + + +INDEPENDENT_BLOCKER_ORDER = ( + "provenance", + "price_lineage", + "temporal_evidence", + "exact_source_rights", + "registered_field_scope", + "unclassified_evidence_blocker", +) +FEATURE_ORDER = {"fundamentals": 0, "dcf": 1} +TEMPORAL_BLOCKERS = { + "invalid_observation_date", + "missing_retrieved_at", + "invalid_retrieved_at", + "retrieved_at_timezone_required", + "review_cutoff_required", + "invalid_review_cutoff", + "review_cutoff_timezone_required", + "retrieved_before_observation_available", + "retrieved_after_review_cutoff", +} + + +@dataclass(frozen=True) +class ReadinessRemediationCandidate: + ticker: str + feature: str + status: str + independent_blockers: tuple[str, ...] + blocker_details: tuple[str, ...] + fundamentals_source: str + rights_status: str + missing_provenance_fields: tuple[str, ...] + missing_registered_fields: tuple[str, ...] + price_source: str + price_rights_status: str + price_temporal_status: str + price_missing_provenance_fields: tuple[str, ...] + price_missing_registered_fields: tuple[str, ...] + next_review_instruction: str + + +@dataclass(frozen=True) +class ReadinessEvidenceRemediation: + status: str + preview_status: str + saved_snapshot_identity: str + proposed_snapshot_identity: str + saved_ticker_count: int + proposed_ticker_count: int + changed_ticker_count: int + added_ticker_count: int + removed_ticker_count: int + method_fit_exclusion_counts: tuple[tuple[str, int], ...] + fundamentals_promotion_count: int + dcf_promotion_count: int + independent_blocker_counts: tuple[tuple[str, int], ...] + candidate_count: int + candidates: tuple[ReadinessRemediationCandidate, ...] + top_n: int + canonical_apply_authorized: bool = False + readiness_materialization_authorized: bool = False + source_rights_change_authorized: bool = False + repository_writes: tuple[str, ...] = () + + +def _independent_blockers( + fundamentals: ReadinessPromotionEvidence, + price: DcfPriceLineageEvidence | None, +) -> tuple[str, ...]: + blockers: set[str] = set() + if fundamentals.missing_provenance_fields: + blockers.add("provenance") + if fundamentals.missing_supported_fields: + blockers.add("registered_field_scope") + for blocker in fundamentals.blockers: + if blocker.startswith("missing_provenance:") or blocker in { + "missing_fundamentals_row", + "duplicate_fundamentals_rows", + }: + blockers.add("provenance") + elif blocker.startswith("commercial_rights:"): + blockers.add("exact_source_rights") + elif blocker == "registered_field_scope_incomplete": + blockers.add("registered_field_scope") + else: + blockers.add("unclassified_evidence_blocker") + + if price is not None: + if price.latest_row_count != 1 or price.missing_provenance_fields: + blockers.add("price_lineage") + if price.latest_row_count == 1 and price.temporal_status != "temporal_complete": + blockers.add("temporal_evidence") + if price.missing_supported_fields: + blockers.add("registered_field_scope") + for blocker in price.blockers: + if blocker.startswith("missing_provenance:") or blocker in { + "missing_latest_price_row", + "ambiguous_latest_price_row", + }: + blockers.add("price_lineage") + elif blocker in TEMPORAL_BLOCKERS: + blockers.add("temporal_evidence") + elif blocker.startswith("commercial_rights:"): + blockers.add("exact_source_rights") + elif blocker == "registered_price_scope_incomplete": + blockers.add("registered_field_scope") + else: + blockers.add("unclassified_evidence_blocker") + return tuple(name for name in INDEPENDENT_BLOCKER_ORDER if name in blockers) + + +def _next_review_instruction( + ticker: str, + blockers: tuple[str, ...], + fundamentals: ReadinessPromotionEvidence, + price: DcfPriceLineageEvidence | None, +) -> str: + if "unclassified_evidence_blocker" in blockers: + details = list(fundamentals.blockers) + if price is not None: + details.extend(price.blockers) + return f"Independent review must classify unexpected evidence blockers for {ticker}: {', '.join(details)}." + if "provenance" in blockers: + fields = ", ".join(fundamentals.missing_provenance_fields) or "one unambiguous fundamentals row" + return f"Review exact fundamentals provenance for {ticker}: {fields}; keep missing evidence uninferred." + if "price_lineage" in blockers: + fields = ", ".join(price.missing_provenance_fields) if price is not None else "latest price evidence" + return f"Review exact latest-price lineage for {ticker}: {fields or 'one unambiguous latest row'}; keep provider identity uninferred." + if "temporal_evidence" in blockers: + return f"Review the exact price retrieval timestamp and cutoff evidence for {ticker}." + if "exact_source_rights" in blockers: + sources: list[str] = [] + if fundamentals.rights_status != "approved": + sources.append(fundamentals.source_id) + if ( + price is not None + and price.rights_status != "approved" + and price.source_id not in {"", "", ""} + ): + sources.append(price.source_id) + return f"Owner decision required for exact-source commercial rights: {' | '.join(sources)}; keep identifiers intact." + if "registered_field_scope" in blockers: + fields = list(fundamentals.missing_supported_fields) + if price is not None: + fields.extend(price.missing_supported_fields) + return f"Owner review required for registered field scope on {ticker}: {', '.join(dict.fromkeys(fields))}." + return f"Evidence gates are complete for {ticker}; independent review is still required before a separate readiness decision." + + +def _candidate( + fundamentals: ReadinessPromotionEvidence, + feature: str, + price: DcfPriceLineageEvidence | None, +) -> ReadinessRemediationCandidate: + blockers = _independent_blockers(fundamentals, price) + details = tuple(f"fundamentals:{item}" for item in fundamentals.blockers) + if price is not None: + details += tuple(f"price:{item}" for item in price.blockers) + return ReadinessRemediationCandidate( + ticker=fundamentals.ticker, + feature=feature, + status="withheld" if blockers else "evidence_reviewable", + independent_blockers=blockers, + blocker_details=details, + fundamentals_source=fundamentals.source_id, + rights_status=fundamentals.rights_status, + missing_provenance_fields=fundamentals.missing_provenance_fields, + missing_registered_fields=fundamentals.missing_supported_fields, + price_source=price.source_id if price is not None else "not_applicable", + price_rights_status=price.rights_status if price is not None else "not_applicable", + price_temporal_status=price.temporal_status if price is not None else "not_applicable", + price_missing_provenance_fields=(price.missing_provenance_fields if price is not None else ()), + price_missing_registered_fields=(price.missing_supported_fields if price is not None else ()), + next_review_instruction=_next_review_instruction(fundamentals.ticker, blockers, fundamentals, price), + ) + + +def build_remediation_from_preview( + preview: ReadinessImpactPreview, + *, + top_n: int = 20, +) -> ReadinessEvidenceRemediation: + """Project existing preview evidence into a closest-first, fail-closed queue.""" + + if top_n < 1: + raise ValueError("top_n must be at least 1") + promotion_review = preview.promotion_review + change_review = preview.change_review + price_review = preview.dcf_price_lineage_review + promotion_rows = promotion_review.evidence_rows if promotion_review is not None else () + price_by_ticker = { + item.ticker: item for item in (price_review.evidence_rows if price_review is not None else ()) + } + all_candidates: list[ReadinessRemediationCandidate] = [] + for evidence in promotion_rows: + for promoted_field in evidence.promoted_fields: + feature = promoted_field.removesuffix("_ready") + price = price_by_ticker.get(evidence.ticker) if feature == "dcf" else None + all_candidates.append(_candidate(evidence, feature, price)) + all_candidates.sort( + key=lambda item: ( + len(item.independent_blockers), + len(item.blocker_details), + item.ticker, + FEATURE_ORDER.get(item.feature, 99), + ) + ) + blocker_counts = tuple( + (name, sum(name in item.independent_blockers for item in all_candidates)) + for name in INDEPENDENT_BLOCKER_ORDER + if any(name in item.independent_blockers for item in all_candidates) + ) + return ReadinessEvidenceRemediation( + status="inspection_only", + preview_status=preview.status, + saved_snapshot_identity=preview.saved_snapshot_identity, + proposed_snapshot_identity=preview.proposed_snapshot_identity, + saved_ticker_count=preview.saved_ticker_count, + proposed_ticker_count=preview.proposed_ticker_count, + changed_ticker_count=preview.changed_ticker_count, + added_ticker_count=change_review.added_ticker_count if change_review is not None else 0, + removed_ticker_count=change_review.removed_ticker_count if change_review is not None else 0, + method_fit_exclusion_counts=(change_review.newly_excluded_counts if change_review is not None else ()), + fundamentals_promotion_count=( + promotion_review.fundamentals_promotion_count if promotion_review is not None else 0 + ), + dcf_promotion_count=promotion_review.dcf_promotion_count if promotion_review is not None else 0, + independent_blocker_counts=blocker_counts, + candidate_count=len(all_candidates), + candidates=tuple(all_candidates[:top_n]), + top_n=top_n, + ) + + +def build_readiness_evidence_remediation( + root: Path | str, + *, + data_dir: Path | str | None = None, + top_n: int = 20, + rights_registry: Mapping[str, SourceRights] | None = None, + review_cutoff: str | None = None, +) -> ReadinessEvidenceRemediation: + preview = build_readiness_impact_preview( + root, + data_dir=data_dir, + top_n=top_n, + rights_registry=rights_registry, + review_cutoff=review_cutoff, + include_all_evidence=True, + ) + return build_remediation_from_preview(preview, top_n=top_n) + + +def render_readiness_evidence_remediation_json(packet: ReadinessEvidenceRemediation) -> str: + return json.dumps(asdict(packet), ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _format_counts(values: tuple[tuple[str, int], ...]) -> str: + return ", ".join(f"{name}={count}" for name, count in values) or "none" + + +def render_readiness_evidence_remediation(packet: ReadinessEvidenceRemediation) -> str: + lines = [ + "Readiness Evidence Remediation Queue", + "", + f"Status: {packet.status}", + f"Preview status: {packet.preview_status}", + f"Saved snapshot identity: {packet.saved_snapshot_identity or ''}", + f"Proposed snapshot identity: {packet.proposed_snapshot_identity or ''}", + f"Ticker rows: saved={packet.saved_ticker_count}, proposed={packet.proposed_ticker_count}", + f"Changed tickers: {packet.changed_ticker_count}", + f"Universe rows: added={packet.added_ticker_count}, removed={packet.removed_ticker_count}", + f"Method-fit exclusions: {_format_counts(packet.method_fit_exclusion_counts)}", + ( + "Technical promotions: " + f"fundamentals={packet.fundamentals_promotion_count}, DCF={packet.dcf_promotion_count}" + ), + f"Independent blocker counts: {_format_counts(packet.independent_blocker_counts)}", + f"Candidates: total={packet.candidate_count}, shown={len(packet.candidates)}, TOP_N={packet.top_n}", + ] + for item in packet.candidates: + blockers = ",".join(item.independent_blockers) or "none" + details = ",".join(item.blocker_details) or "none" + missing_scope = ",".join(item.missing_registered_fields) or "none" + missing_provenance = ",".join(item.missing_provenance_fields) or "none" + price_missing_scope = ",".join(item.price_missing_registered_fields) or "none" + price_missing_provenance = ",".join(item.price_missing_provenance_fields) or "none" + lines.extend( + [ + ( + f"- {item.ticker} / {item.feature}: status={item.status}; blockers={blockers}; " + f"details={details}" + ), + ( + f" fundamentals_source={item.fundamentals_source!r}; rights={item.rights_status}; " + f"missing_provenance={missing_provenance}; missing_registered_fields={missing_scope}" + ), + ( + f" price_source={item.price_source!r}; price_rights={item.price_rights_status}; " + f"price_temporal={item.price_temporal_status}; " + f"price_missing_provenance={price_missing_provenance}; " + f"price_missing_registered_fields={price_missing_scope}" + ), + f" next_review={item.next_review_instruction}", + ] + ) + lines.extend( + [ + "", + f"canonical_apply_authorized={str(packet.canonical_apply_authorized).lower()}", + ( + "readiness_materialization_authorized=" + f"{str(packet.readiness_materialization_authorized).lower()}" + ), + f"source_rights_change_authorized={str(packet.source_rights_change_authorized).lower()}", + "repository_writes=[]", + "Numerical completeness never overrides an independent evidence blocker.", + "Research workflow evidence only; not investment advice or a recommendation.", + ] + ) + return "\n".join(lines) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Build a stdout-only readiness evidence remediation queue.") + parser.add_argument("--project-root", default=".") + parser.add_argument("--data-dir") + parser.add_argument("--top-n", type=int, default=20) + parser.add_argument("--review-cutoff") + parser.add_argument("--json", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + packet = build_readiness_evidence_remediation( + Path(args.project_root), + data_dir=args.data_dir, + top_n=args.top_n, + review_cutoff=args.review_cutoff, + ) + except (KeyError, OSError, ValueError) as exc: + print(f"Readiness evidence remediation failed: {exc}") + print("repository_writes=[]") + return 1 + print( + render_readiness_evidence_remediation_json(packet) + if args.json + else render_readiness_evidence_remediation(packet) + ) + return 2 if packet.preview_status == "missing_saved_snapshot" else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/readiness_preview.py b/src/readiness_preview.py index 8b6a8e05..56c0f4f3 100644 --- a/src/readiness_preview.py +++ b/src/readiness_preview.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import hashlib +import json from dataclasses import dataclass, replace from pathlib import Path from typing import Mapping @@ -121,6 +123,8 @@ class ReadinessImpactPreview: promotion_review: ReadinessPromotionReview | None = None change_review: ReadinessChangeReview | None = None dcf_price_lineage_review: DcfPriceLineageReview | None = None + saved_snapshot_identity: str = "" + proposed_snapshot_identity: str = "" def _truthy(value: object) -> bool: @@ -143,6 +147,36 @@ def _stable_value(row: pd.Series, field: str) -> bool | str: return _text(row.get(field)) +def _snapshot_identity(frame: pd.DataFrame) -> str: + columns = [ + column + for column in ("ticker", *STABLE_READINESS_FIELDS) + if column in frame.columns + ] + normalized = frame.reindex(columns=columns) + rows: list[list[object]] = [] + for values in normalized.itertuples(index=False, name=None): + row: list[object] = [] + for value in values: + if pd.isna(value): + row.append(None) + elif isinstance(value, bool): + row.append(value) + elif isinstance(value, (int, float)): + row.append(value) + else: + row.append(str(value)) + rows.append(row) + rows.sort(key=lambda item: json.dumps(item, ensure_ascii=False, separators=(",", ":"))) + payload = json.dumps( + {"columns": columns, "rows": rows}, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + def _index_readiness(frame: pd.DataFrame) -> dict[str, pd.Series]: if frame.empty or "ticker" not in frame.columns: return {} @@ -424,6 +458,8 @@ def compare_readiness_frames( changed_tickers=tuple(changes[:top_n]), top_n=top_n, saved_path=saved_path, + saved_snapshot_identity=_snapshot_identity(saved), + proposed_snapshot_identity=_snapshot_identity(proposed), ) @@ -434,6 +470,7 @@ def build_readiness_impact_preview( top_n: int = 20, rights_registry: Mapping[str, SourceRights] | None = None, review_cutoff: str | None = None, + include_all_evidence: bool = False, ) -> ReadinessImpactPreview: project_root = resolve_project_root(root) data_path = resolve_data_dir(data_dir, project_root) @@ -457,6 +494,7 @@ def build_readiness_impact_preview( write_outputs=False, ) proposed = reports["ticker_readiness_report"] + evidence_top_n = max(top_n, len(proposed)) if include_all_evidence else top_n preview = compare_readiness_frames( saved, proposed, @@ -477,7 +515,7 @@ def build_readiness_impact_preview( proposed, fundamentals, rights_registry=registry, - top_n=top_n, + top_n=evidence_top_n, ) change_review = review_readiness_changes(saved, proposed, fundamentals) prices_path = data_path / "prices.csv" @@ -490,7 +528,7 @@ def build_readiness_impact_preview( prices, rights_registry=registry, review_cutoff=review_cutoff, - top_n=top_n, + top_n=evidence_top_n, ) return replace( preview, diff --git a/tests/test_launchers.py b/tests/test_launchers.py index e3fa3503..67a2fea6 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -132,6 +132,65 @@ def test_readiness_release_review_make_is_json_and_write_free(): assert after_status == before_status +def test_readiness_evidence_remediation_make_is_deterministic_json_and_write_free(): + root = Path.cwd() + before = _tree_manifest(root) + + first = subprocess.run( + ["make", "--no-print-directory", "readiness-evidence-remediation", "TOP_N=2", "JSON=1"], + cwd=root, + capture_output=True, + text=True, + check=False, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + second = subprocess.run( + ["make", "--no-print-directory", "readiness-evidence-remediation", "TOP_N=2", "JSON=1"], + cwd=root, + capture_output=True, + text=True, + check=False, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + + assert first.returncode == 0, first.stderr + assert second.returncode == 0, second.stderr + assert first.stdout == second.stdout + payload = json.loads(first.stdout) + assert payload["status"] == "inspection_only" + assert payload["canonical_apply_authorized"] is False + assert payload["readiness_materialization_authorized"] is False + assert payload["source_rights_change_authorized"] is False + assert payload["repository_writes"] == [] + assert _tree_manifest(root) == before + + +def test_interview_brief_local_artifact_boundary_is_narrow(): + expected_local = ( + "output/documents/Stock_Research_Command_Center_Interview_Brief.docx", + "output/pdf/Stock_Research_Command_Center_Interview_Brief.pdf", + "output/source/Stock_Research_Command_Center_Interview_Brief.py", + "tmp/interview_brief/render/page-1.png", + ) + ignored = subprocess.run( + ["git", "check-ignore", "--no-index", *expected_local], + capture_output=True, + text=True, + check=False, + ) + unrelated = subprocess.run( + ["git", "check-ignore", "--no-index", "output/unrelated.txt", "tmp/unrelated.txt"], + capture_output=True, + text=True, + check=False, + ) + + assert ignored.returncode == 0 + assert ignored.stdout.splitlines() == list(expected_local) + assert unrelated.returncode == 1 + assert unrelated.stdout == "" + + def test_make_reachability_tracks_every_target_on_recursive_multi_target_lines(): makefile = Path("Makefile").read_text(encoding="utf-8") diff --git a/tests/test_readiness_evidence_remediation.py b/tests/test_readiness_evidence_remediation.py new file mode 100644 index 00000000..2ed24e9d --- /dev/null +++ b/tests/test_readiness_evidence_remediation.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path + +import pandas as pd + +from src import readiness_preview +from src.commercial_source_rights import SourceRights +from src.dcf_price_lineage import review_dcf_price_lineage +from src.readiness_evidence_remediation import ( + build_readiness_evidence_remediation, + build_remediation_from_preview, + render_readiness_evidence_remediation, + render_readiness_evidence_remediation_json, +) +from src.readiness_preview import ( + compare_readiness_frames, + review_readiness_changes, + review_readiness_promotions, +) + + +def _readiness_row(ticker: str, **overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "ticker": ticker, + "name": f"{ticker} Software", + "asset_type": "company", + "overall_readiness_state": "blocked", + "price_ready": False, + "momentum_ready": False, + "fundamentals_ready": False, + "dcf_ready": False, + "peer_ready": False, + "earnings_ready": False, + "analyst_estimates_ready": False, + "ready_features": "", + "partial_features": "", + "blocked_features": "fundamentals, dcf", + "excluded_features": "", + } + row.update(overrides) + return row + + +def _rights( + source_id: str, + *, + commercial_use: str = "approved", + supported_fields: tuple[str, ...], +) -> SourceRights: + return SourceRights( + source_id=source_id, + display_name=source_id, + permitted_use="source_backed_research", + commercial_use=commercial_use, + redistribution="derived_data_only", + storage_limits="reviewed local rows", + attribution="required", + rate_limits="provider terms", + authentication="provider specific", + expected_freshness="event driven", + supported_fields=supported_fields, + fallback_priority=1, + ) + + +def _registry() -> dict[str, SourceRights]: + return { + "full_fundamentals": _rights( + "full_fundamentals", + supported_fields=("revenue", "free_cash_flow", "fcf_margin", "shares_outstanding"), + ), + "scope_gap": _rights("scope_gap", supported_fields=("revenue",)), + "approved_prices": _rights("approved_prices", supported_fields=("prices",)), + } + + +def _promotion_fixture(): + saved = pd.DataFrame( + [ + _readiness_row("AAA"), + _readiness_row("BBB"), + _readiness_row("CCC"), + _readiness_row("DDD"), + _readiness_row("EEE"), + _readiness_row("EXC"), + ] + ) + proposed = pd.DataFrame( + [ + _readiness_row("AAA", fundamentals_ready=True, dcf_ready=True), + _readiness_row("BBB", fundamentals_ready=True), + _readiness_row("CCC", fundamentals_ready=True, dcf_ready=True), + _readiness_row("DDD", fundamentals_ready=True, dcf_ready=True), + _readiness_row("EEE", fundamentals_ready=True), + _readiness_row( + "EXC", + name="Example Bank Corp", + excluded_features="dcf", + blocked_features="fundamentals", + ), + _readiness_row("ADD"), + ] + ) + fundamentals = pd.DataFrame( + [ + { + "ticker": "AAA", + "source": "full_fundamentals; filing_document", + "as_of_date": "2025-12-31", + "source_ref": "filing:AAA", + }, + { + "ticker": "BBB", + "source": "scope_gap", + "as_of_date": "2025-12-31", + "source_ref": "filing:BBB", + }, + { + "ticker": "CCC", + "source": "full_fundamentals", + "as_of_date": "2025-12-31", + "source_ref": "filing:CCC", + }, + { + "ticker": "DDD", + "source": "full_fundamentals", + "as_of_date": "2025-12-31", + "source_ref": "filing:DDD", + }, + { + "ticker": "EEE", + "source": "full_fundamentals", + "as_of_date": "2025-12-31", + "source_ref": "", + }, + ] + ) + prices = pd.DataFrame( + [ + { + "ticker": "AAA", + "date": "2026-01-03", + "close": 10, + "source": "approved_prices", + "source_ref": "price:AAA", + "retrieved_at": "2026-01-04T23:00:00Z", + }, + { + "ticker": "CCC", + "date": "2026-01-03", + "close": 10, + "source": "approved_prices", + "source_ref": "price:CCC", + "retrieved_at": "2026-01-04T23:00:00", + }, + { + "ticker": "DDD", + "date": "2026-01-03", + "close": 10, + "source": "approved_prices", + "source_ref": "price:DDD:1", + "retrieved_at": "2026-01-04T23:00:00Z", + }, + { + "ticker": "DDD", + "date": "2026-01-03", + "close": 11, + "source": "approved_prices", + "source_ref": "price:DDD:2", + "retrieved_at": "2026-01-04T23:00:00Z", + }, + ] + ) + preview = compare_readiness_frames(saved, proposed, top_n=20) + return replace( + preview, + promotion_review=review_readiness_promotions( + saved, + proposed, + fundamentals, + rights_registry=_registry(), + top_n=20, + ), + change_review=review_readiness_changes(saved, proposed, fundamentals), + dcf_price_lineage_review=review_dcf_price_lineage( + saved, + proposed, + prices, + rights_registry=_registry(), + review_cutoff="2026-01-05T00:00:00Z", + top_n=20, + ), + ) + + +def _tree_manifest(root: Path) -> dict[str, bytes]: + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def test_queue_combines_independent_fail_closed_blockers_without_splitting_sources(): + packet = build_remediation_from_preview(_promotion_fixture(), top_n=20) + by_key = {(item.ticker, item.feature): item for item in packet.candidates} + + assert packet.status == "inspection_only" + assert packet.changed_ticker_count == 7 + assert packet.added_ticker_count == 1 + assert packet.removed_ticker_count == 0 + assert packet.fundamentals_promotion_count == 5 + assert packet.dcf_promotion_count == 3 + assert packet.method_fit_exclusion_counts == (("dcf", 1),) + assert packet.canonical_apply_authorized is False + assert packet.readiness_materialization_authorized is False + assert packet.source_rights_change_authorized is False + assert packet.repository_writes == () + + composite = by_key[("AAA", "fundamentals")] + assert composite.fundamentals_source == "full_fundamentals; filing_document" + assert composite.rights_status == "unknown_source" + assert composite.status == "withheld" + assert "exact_source_rights" in composite.independent_blockers + assert "registered_field_scope" in composite.independent_blockers + + scope_gap = by_key[("BBB", "fundamentals")] + assert scope_gap.rights_status == "approved" + assert scope_gap.missing_registered_fields == ( + "free_cash_flow", + "fcf_margin", + "shares_outstanding", + ) + assert scope_gap.independent_blockers == ("registered_field_scope",) + + temporal = by_key[("CCC", "dcf")] + assert temporal.price_temporal_status == "temporal_review_required" + assert "temporal_evidence" in temporal.independent_blockers + + ambiguous = by_key[("DDD", "dcf")] + assert ambiguous.price_source == "" + assert "price_lineage" in ambiguous.independent_blockers + + provenance = by_key[("EEE", "fundamentals")] + assert provenance.missing_provenance_fields == ("source_reference",) + assert provenance.independent_blockers == ("provenance",) + + +def test_queue_is_closest_first_capped_and_json_is_byte_identical(): + preview = _promotion_fixture() + first = build_remediation_from_preview(preview, top_n=3) + second = build_remediation_from_preview(preview, top_n=3) + + assert first.candidate_count == 8 + assert len(first.candidates) == 3 + assert [(item.ticker, item.feature) for item in first.candidates] == [ + ("CCC", "fundamentals"), + ("DDD", "fundamentals"), + ("BBB", "fundamentals"), + ] + assert render_readiness_evidence_remediation_json(first) == render_readiness_evidence_remediation_json(second) + payload = json.loads(render_readiness_evidence_remediation_json(first)) + assert payload["repository_writes"] == [] + assert payload["candidates"][0]["next_review_instruction"] + + +def test_queue_fails_closed_on_an_unclassified_upstream_blocker(): + preview = _promotion_fixture() + assert preview.promotion_review is not None + evidence_rows = tuple( + replace(item, blockers=("future_evidence_gate:blocked",)) + if item.ticker == "CCC" + else item + for item in preview.promotion_review.evidence_rows + ) + preview = replace( + preview, + promotion_review=replace(preview.promotion_review, evidence_rows=evidence_rows), + ) + + packet = build_remediation_from_preview(preview, top_n=20) + candidate = next( + item for item in packet.candidates if item.ticker == "CCC" and item.feature == "fundamentals" + ) + + assert candidate.status == "withheld" + assert "unclassified_evidence_blocker" in candidate.independent_blockers + assert "fundamentals:future_evidence_gate:blocked" in candidate.blocker_details + + +def test_rights_instruction_names_only_sources_with_unresolved_rights(): + preview = _promotion_fixture() + fundamentals_blocked = build_remediation_from_preview(preview, top_n=20) + aaa_dcf = next( + item + for item in fundamentals_blocked.candidates + if item.ticker == "AAA" and item.feature == "dcf" + ) + + assert "full_fundamentals; filing_document" in aaa_dcf.next_review_instruction + assert "approved_prices" not in aaa_dcf.next_review_instruction + + assert preview.dcf_price_lineage_review is not None + price_rows = tuple( + replace( + item, + source_id="unknown_prices", + rights_status="unknown_source", + temporal_status="temporal_complete", + retrieved_at="2026-01-04T23:00:00+00:00", + missing_provenance_fields=(), + blockers=("commercial_rights:unknown_source",), + ) + if item.ticker == "CCC" + else item + for item in preview.dcf_price_lineage_review.evidence_rows + ) + price_blocked_preview = replace( + preview, + dcf_price_lineage_review=replace( + preview.dcf_price_lineage_review, + evidence_rows=price_rows, + ), + ) + price_blocked = build_remediation_from_preview(price_blocked_preview, top_n=20) + ccc_dcf = next( + item + for item in price_blocked.candidates + if item.ticker == "CCC" and item.feature == "dcf" + ) + + assert "unknown_prices" in ccc_dcf.next_review_instruction + assert "full_fundamentals" not in ccc_dcf.next_review_instruction + + +def test_queue_empty_set_and_text_preserve_inspection_only_boundary(): + saved = pd.DataFrame([_readiness_row("AAA")]) + preview = compare_readiness_frames(saved, saved.copy(), top_n=5) + preview = replace( + preview, + promotion_review=review_readiness_promotions( + saved, + saved.copy(), + pd.DataFrame(), + rights_registry={}, + top_n=5, + ), + change_review=review_readiness_changes(saved, saved.copy(), pd.DataFrame()), + dcf_price_lineage_review=review_dcf_price_lineage( + saved, + saved.copy(), + pd.DataFrame(), + rights_registry={}, + top_n=5, + ), + ) + + packet = build_remediation_from_preview(preview, top_n=5) + rendered = render_readiness_evidence_remediation(packet) + + assert packet.candidate_count == 0 + assert packet.candidates == () + assert "Status: inspection_only" in rendered + assert "canonical_apply_authorized=false" in rendered + assert "readiness_materialization_authorized=false" in rendered + assert "source_rights_change_authorized=false" in rendered + assert "repository_writes=[]" in rendered + lowered = rendered.lower() + for unsafe in ("buy", "sell", "order routing", "auto-trading"): + assert unsafe not in lowered + + +def test_build_uses_full_existing_preview_evidence_and_writes_nothing(tmp_path: Path, monkeypatch): + reports = tmp_path / "data" / "reports" + reports.mkdir(parents=True) + saved = pd.DataFrame([_readiness_row("AAA")]) + proposed = pd.DataFrame([_readiness_row("AAA", fundamentals_ready=True)]) + saved.to_csv(reports / "ticker_readiness_report.csv", index=False) + pd.DataFrame( + [ + { + "ticker": "AAA", + "source": "full_fundamentals", + "as_of_date": "2025-12-31", + "source_ref": "filing:AAA", + } + ] + ).to_csv(tmp_path / "data" / "fundamentals.csv", index=False) + + def _build(*args: object, **kwargs: object) -> dict[str, pd.DataFrame]: + assert kwargs["write_outputs"] is False + return {"ticker_readiness_report": proposed} + + monkeypatch.setattr(readiness_preview, "build_ticker_readiness_report", _build) + before = _tree_manifest(tmp_path) + + packet = build_readiness_evidence_remediation( + tmp_path, + top_n=1, + rights_registry=_registry(), + review_cutoff="2026-01-05T00:00:00Z", + ) + + assert packet.saved_snapshot_identity.startswith("sha256:") + assert packet.proposed_snapshot_identity.startswith("sha256:") + assert packet.candidate_count == 1 + assert _tree_manifest(tmp_path) == before diff --git a/tests/test_readiness_preview.py b/tests/test_readiness_preview.py index 1755435b..1782f8da 100644 --- a/tests/test_readiness_preview.py +++ b/tests/test_readiness_preview.py @@ -109,6 +109,9 @@ def test_compare_ignores_updated_at_but_reports_stable_fields(): assert preview.status == "changes_detected" assert preview.changed_ticker_count == 1 + assert preview.saved_snapshot_identity.startswith("sha256:") + assert preview.proposed_snapshot_identity.startswith("sha256:") + assert preview.saved_snapshot_identity != preview.proposed_snapshot_identity assert preview.changed_tickers[0].ticker == "AAA" assert preview.changed_tickers[0].fields == ("price_ready", "ready_features", "blocked_features")