diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff99b78..54c7289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -673,20 +673,29 @@ jobs: fi fi } - require test "$TEST_RESULT" - require py-compat "$COMPAT_RESULT" + # AXW-003A: gate on the GatePlan semantic IDs, not GitHub job names. + # GatePlan emits `py-primary` for the OS/KB/integration suite (its + # result arrives via the `test` job) and `static` for convention and + # architecture checks (carried by the `lint` job). Requiring the bare + # job name `test` would never match GatePlan and would let a required + # py-primary failure pass as green. + require py-primary "$TEST_RESULT" + require static "$LINT_RESULT" require lint "$LINT_RESULT" + require py-compat "$COMPAT_RESULT" require wheel-smoke "$WHEEL_RESULT" require browser-smoke "$BROWSER_RESULT" require windows-runtime "$WINDOWS_RESULT" require desktop-fast "$DESKTOP_FAST_RESULT" require desktop-build "$DESKTOP_BUILD_RESULT" require installer-lifecycle "$INSTALLER_RESULT" - # A not-required job that RAN and failed is still a failure. - for spec in "wheel-smoke:$WHEEL_RESULT" "browser-smoke:$BROWSER_RESULT" "windows-runtime:$WINDOWS_RESULT" "desktop-fast:$DESKTOP_FAST_RESULT" "desktop-build:$DESKTOP_BUILD_RESULT" "installer-lifecycle:$INSTALLER_RESULT"; do + # A not-required job that RAN and failed is still a failure. Use the + # job names (the only labels the `needs.*.result` are keyed by) but + # check them against their real job results. + for spec in "test:$TEST_RESULT" "py-compat:$COMPAT_RESULT" "lint:$LINT_RESULT" "wheel-smoke:$WHEEL_RESULT" "browser-smoke:$BROWSER_RESULT" "windows-runtime-smoke:$WINDOWS_RESULT" "desktop-fast:$DESKTOP_FAST_RESULT" "desktop-build:$DESKTOP_BUILD_RESULT" "installer-lifecycle:$INSTALLER_RESULT"; do name="${spec%%:*}"; result="${spec##*:}" if [ "$result" = "failure" ]; then - echo "gate '$name' failed even though not required" + echo "job '$name' failed even though its gate was not required" exit 1 fi done diff --git a/.worklab/project-validation.v1.yaml b/.worklab/project-validation.v1.yaml index 619b17c..5246669 100644 --- a/.worklab/project-validation.v1.yaml +++ b/.worklab/project-validation.v1.yaml @@ -88,13 +88,22 @@ risk_classes: gates: [static, lint, py-primary, wheel-smoke] - id: python-compat - description: Public Python contracts and dependency matrix + description: Public Python contracts, dependency matrix and requirements paths: - pyproject.toml + - requirements.txt - uv.lock - "shared-contracts/**" gates: [static, lint, py-compat, wheel-smoke] + - id: format-parser + description: Format parsers / conversion engines that affect the installed wheel + paths: + - "app/ingestion/pdf.py" + - "app/ingestion/multi_format.py" + - "app/ingestion/*.py" + gates: [static, lint, py-primary, wheel-smoke] + - id: windows-runtime description: Windows storage, migration, process, path handling paths: diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index e67d539..a62613d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -11,7 +11,9 @@ The release dependency contract is `pyproject.toml` plus the exact resolved `fastapi`, `python-multipart`, `uvicorn`, `pydantic`, `numpy`, `requests`, `pyyaml`, `beautifulsoup4`, `defusedxml`, `apscheduler`, `sqlite-vec`, `loguru`, -`structlog`, `markitdown`, `trafilatura`, `networkx`, `litellm`, `pillow`, and +`structlog`, `markitdown[pdf]` (with `pdfminer-six`, `pdfplumber`, and +`pypdfium2` for PDF extraction), `trafilatura`, `networkx`, `litellm`, +`pillow`, and `pytesseract`. Optional or development groups additionally declare `setuptools`, diff --git a/app/ingestion/raw_asset.py b/app/ingestion/raw_asset.py new file mode 100644 index 0000000..250a535 --- /dev/null +++ b/app/ingestion/raw_asset.py @@ -0,0 +1,140 @@ +"""AXW-012A: RawAsset-first minimal store. + +Contract: original bytes are persisted immutably (content-addressed by +SHA-256) BEFORE any conversion runs; a failed conversion must still retain the +original plus a durable failure record. Failure injection must never lose the +original. + +The default storage root lives under the project's ignored `.hermes/` +runtime boundary; it never touches the source vault or a tracked path. +""" +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + + +class RawAssetStoreError(ValueError): + """Raised on invalid input or an unrecoverable storage failure.""" + + +def _sha256(b: bytes) -> str: + return hashlib.sha256(b).hexdigest() + + +def _default_root() -> Path: + # Prefer HERMES_PROJECT_RUNTIME_ROOT if provided by the project data + # wrapper; otherwise fall back to the repository .hermes/task-runtime. + env_root = os.environ.get("HERMES_PROJECT_RUNTIME_ROOT") + if env_root: + return Path(env_root) / "raw-assets" + # Repository root = /app/ingestion/raw_asset.py -> parents[2] + repo_root = Path(__file__).resolve().parents[2] + return repo_root / ".hermes" / "task-runtime" / "raw-assets" + + +@dataclass(frozen=True) +class RawAssetRecord: + sha256: str + size_bytes: int + source_name: str + converted: str | None + error: str | None = None + + @property + def original_sha(self) -> str: + return self.sha256 + + +class RawAssetStore: + """Content-addressed immutable store for original source bytes.""" + + def __init__(self, root: Path | None = None) -> None: + self.root = (root or _default_root()).resolve() + self.root.mkdir(parents=True, exist_ok=True) + self._failures_dir = self.root / "_failures" + self._failures_dir.mkdir(parents=True, exist_ok=True) + + def _original_path(self, digest: str) -> Path: + return self.root / digest + + def _failure_path(self, digest: str) -> Path: + return self._failures_dir / f"{digest}.json" + + def has(self, digest: str) -> bool: + return self._original_path(digest).exists() + + def has_failure(self, digest: str) -> bool: + return self._failure_path(digest).exists() + + def resolve(self, digest: str) -> Path: + p = self._original_path(digest) + if not p.exists(): + raise RawAssetStoreError(f"raw asset not present: {digest}") + return p + + def store_original(self, blob: bytes, source_name: str) -> RawAssetRecord: + """Persist the original bytes immutably and return a record. Raises on + empty input so empty content can never masquerade as a source asset.""" + if not source_name.strip(): + raise RawAssetStoreError("source_name is required") + if not blob: + raise RawAssetStoreError("empty original bytes cannot be stored") + digest = _sha256(blob) + dest = self._original_path(digest) + # Immutable write: only write when the content-addressed file is absent, + # and verify the hash after writing (no silent partial/corrupt writes). + if not dest.exists(): + dest.write_bytes(blob) + if _sha256(dest.read_bytes()) != digest: + raise RawAssetStoreError("raw asset hash mismatch after write") + return RawAssetRecord( + sha256=digest, + size_bytes=len(blob), + source_name=source_name, + converted=None, + ) + + def _record_failure(self, digest: str, source_name: str, error: str) -> None: + payload = { + "sha256": digest, + "source_name": source_name, + "error": error, + "original_retained": True, + } + fp = self._failure_path(digest) + if not fp.exists(): + fp.write_text(json.dumps(payload, ensure_ascii=True, indent=2), encoding="utf-8") + + +def preserve_then_convert( + store: RawAssetStore, + blob: bytes, + source_name: str, + convert: Callable[[bytes], str], +) -> RawAssetRecord: + """Persist the original first, then convert. On any converter failure the + original is retained and a durable failure record is written. Returns a + record whose `converted` is None and `error` populated on failure.""" + original = store.store_original(blob, source_name) + try: + converted = convert(blob) + except BaseException as exc: # noqa: BLE001 — we must not lose the original + store._record_failure(original.sha256, source_name, str(exc)) + return RawAssetRecord( + sha256=original.sha256, + size_bytes=original.size_bytes, + source_name=source_name, + converted=None, + error=str(exc), + ) + return RawAssetRecord( + sha256=original.sha256, + size_bytes=original.size_bytes, + source_name=source_name, + converted=converted, + ) diff --git a/app/release-manifest.json b/app/release-manifest.json index 2d1eca1..49742f1 100644 --- a/app/release-manifest.json +++ b/app/release-manifest.json @@ -24,9 +24,9 @@ "dependency_lock": { "path": "uv.lock", "algorithm": "sha256", - "digest": "e103c5f9a46ca2e11d50460b610de648978b62e60f53bdd7dd97d81fcf121cf8", + "digest": "9916e6dba6d152cff045abf188218a0a3a533a786b3c3061738dc3a128aa4268", "format_version": 1, - "revision": 4 + "revision": 5 }, "migrations": { "owners": [ diff --git a/pyproject.toml b/pyproject.toml index 61e2f79..4c2f508 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "sqlite-vec>=0.1.6", "loguru>=0.7", "structlog>=24.0", - "markitdown>=0.1", + "markitdown[pdf]>=0.1", "trafilatura>=1.6", "networkx>=3.0", "litellm==1.91.0", @@ -61,7 +61,7 @@ ci = [ "uvicorn[standard]>=0.22", ] ci-adapters = [ - "markitdown>=0.1", + "markitdown[pdf]>=0.1", "newspaper4k>=0.9", "readabilipy>=0.3", "trafilatura>=1.6", diff --git a/requirements.txt b/requirements.txt index 4c219a0..e7a02e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ apscheduler>=3.10 sqlite-vec>=0.1.6 loguru>=0.7 structlog>=24.0 -markitdown>=0.1 +markitdown[pdf]>=0.1 trafilatura>=1.6 networkx>=3.0 litellm==1.91.0 diff --git a/scripts/doctor_windows.ps1 b/scripts/doctor_windows.ps1 new file mode 100644 index 0000000..eee771f --- /dev/null +++ b/scripts/doctor_windows.ps1 @@ -0,0 +1,146 @@ +#requires -Version 7.0 +<# + SYNOPSIS + Windows/PowerShell 7 doctor for Cognitive-Loop-OS (AXW-007A). + + DESCRIPTION + Detects the toolchain and Windows-environment prerequisites needed to run, + test, and package the project: Python, Node, Rust, PowerShell, Chinese and + space-containing paths, port availability, console encoding, and writable + directories. + + Output is strictly sanitized: it never prints secrets, tokens, cookies, + credentials, private paths outside the declared scope, or personal body + text. Only names, versions, availability booleans and path-layout facts are + emitted. All results are returned as structured JSON on stdout; warnings go + to stderr. + + OUTPUT + A single JSON object: + { + "schema_version": "axw.007a.v1", + "generated_at": "...", + "toolchain": { "python": {...}, "node": {...}, "rust": {...}, "powershell": {...} }, + "paths": { "space_in_path": bool, "non_ascii_in_path": bool, "project_root": "", ... }, + "ports": { "