H0 v0.5.1 implementation batch (AXW-003A/003C/007A/009B/011A/012A/012B)

.github/workflows/ci.yml

@@ -673,20 +673,29 @@ jobs:
               fi
             fi
           }
-          require test "$TEST_RESULT"
-          require py-compat "$COMPAT_RESULT"
+          # AXW-003A: gate on the GatePlan semantic IDs, not GitHub job names.
+          # GatePlan emits `py-primary` for the OS/KB/integration suite (its
+          # result arrives via the `test` job) and `static` for convention and
+          # architecture checks (carried by the `lint` job). Requiring the bare
+          # job name `test` would never match GatePlan and would let a required
+          # py-primary failure pass as green.
+          require py-primary "$TEST_RESULT"
+          require static "$LINT_RESULT"
           require lint "$LINT_RESULT"
+          require py-compat "$COMPAT_RESULT"
           require wheel-smoke "$WHEEL_RESULT"
           require browser-smoke "$BROWSER_RESULT"
           require windows-runtime "$WINDOWS_RESULT"
           require desktop-fast "$DESKTOP_FAST_RESULT"
           require desktop-build "$DESKTOP_BUILD_RESULT"
           require installer-lifecycle "$INSTALLER_RESULT"
-          # A not-required job that RAN and failed is still a failure.
-          for spec in "wheel-smoke:$WHEEL_RESULT" "browser-smoke:$BROWSER_RESULT" "windows-runtime:$WINDOWS_RESULT" "desktop-fast:$DESKTOP_FAST_RESULT" "desktop-build:$DESKTOP_BUILD_RESULT" "installer-lifecycle:$INSTALLER_RESULT"; do
+          # A not-required job that RAN and failed is still a failure. Use the
+          # job names (the only labels the `needs.*.result` are keyed by) but
+          # check them against their real job results.
+          for spec in "test:$TEST_RESULT" "py-compat:$COMPAT_RESULT" "lint:$LINT_RESULT" "wheel-smoke:$WHEEL_RESULT" "browser-smoke:$BROWSER_RESULT" "windows-runtime-smoke:$WINDOWS_RESULT" "desktop-fast:$DESKTOP_FAST_RESULT" "desktop-build:$DESKTOP_BUILD_RESULT" "installer-lifecycle:$INSTALLER_RESULT"; do
             name="${spec%%:*}"; result="${spec##*:}"
             if [ "$result" = "failure" ]; then
-              echo "gate '$name' failed even though not required"
+              echo "job '$name' failed even though its gate was not required"
               exit 1
             fi
           done

.worklab/project-validation.v1.yaml

@@ -88,13 +88,22 @@ risk_classes:
     gates: [static, lint, py-primary, wheel-smoke]
 
   - id: python-compat
-    description: Public Python contracts and dependency matrix
+    description: Public Python contracts, dependency matrix and requirements
     paths:
       - pyproject.toml
+      - requirements.txt
       - uv.lock
       - "shared-contracts/**"
     gates: [static, lint, py-compat, wheel-smoke]
 
+  - id: format-parser
+    description: Format parsers / conversion engines that affect the installed wheel
+    paths:
+      - "app/ingestion/pdf.py"
+      - "app/ingestion/multi_format.py"
+      - "app/ingestion/*.py"
+    gates: [static, lint, py-primary, wheel-smoke]
+
   - id: windows-runtime
     description: Windows storage, migration, process, path handling
     paths:

THIRD_PARTY_NOTICES.md

@@ -11,7 +11,9 @@ The release dependency contract is `pyproject.toml` plus the exact resolved
 
 `fastapi`, `python-multipart`, `uvicorn`, `pydantic`, `numpy`, `requests`,
 `pyyaml`, `beautifulsoup4`, `defusedxml`, `apscheduler`, `sqlite-vec`, `loguru`,
-`structlog`, `markitdown`, `trafilatura`, `networkx`, `litellm`, `pillow`, and
+`structlog`, `markitdown[pdf]` (with `pdfminer-six`, `pdfplumber`, and
+`pypdfium2` for PDF extraction), `trafilatura`, `networkx`, `litellm`,
+`pillow`, and
 `pytesseract`.
 
 Optional or development groups additionally declare `setuptools`,

app/ingestion/raw_asset.py

@@ -0,0 +1,140 @@
+"""AXW-012A: RawAsset-first minimal store.
+
+Contract: original bytes are persisted immutably (content-addressed by
+SHA-256) BEFORE any conversion runs; a failed conversion must still retain the
+original plus a durable failure record. Failure injection must never lose the
+original.
+
+The default storage root lives under the project's ignored `.hermes/`
+runtime boundary; it never touches the source vault or a tracked path.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class RawAssetStoreError(ValueError):
+    """Raised on invalid input or an unrecoverable storage failure."""
+
+
+def _sha256(b: bytes) -> str:
+    return hashlib.sha256(b).hexdigest()
+
+
+def _default_root() -> Path:
+    # Prefer HERMES_PROJECT_RUNTIME_ROOT if provided by the project data
+    # wrapper; otherwise fall back to the repository .hermes/task-runtime.
+    env_root = os.environ.get("HERMES_PROJECT_RUNTIME_ROOT")
+    if env_root:
+        return Path(env_root) / "raw-assets"
+    # Repository root = <repo>/app/ingestion/raw_asset.py -> parents[2]
+    repo_root = Path(__file__).resolve().parents[2]
+    return repo_root / ".hermes" / "task-runtime" / "raw-assets"
+
+
+@dataclass(frozen=True)
+class RawAssetRecord:
+    sha256: str
+    size_bytes: int
+    source_name: str
+    converted: str | None
+    error: str | None = None
+
+    @property
+    def original_sha(self) -> str:
+        return self.sha256
+
+
+class RawAssetStore:
+    """Content-addressed immutable store for original source bytes."""
+
+    def __init__(self, root: Path | None = None) -> None:
+        self.root = (root or _default_root()).resolve()
+        self.root.mkdir(parents=True, exist_ok=True)
+        self._failures_dir = self.root / "_failures"
+        self._failures_dir.mkdir(parents=True, exist_ok=True)
+
+    def _original_path(self, digest: str) -> Path:
+        return self.root / digest
+
+    def _failure_path(self, digest: str) -> Path:
+        return self._failures_dir / f"{digest}.json"
+
+    def has(self, digest: str) -> bool:
+        return self._original_path(digest).exists()
+
+    def has_failure(self, digest: str) -> bool:
+        return self._failure_path(digest).exists()
+
+    def resolve(self, digest: str) -> Path:
+        p = self._original_path(digest)
+        if not p.exists():
+            raise RawAssetStoreError(f"raw asset not present: {digest}")
+        return p
+
+    def store_original(self, blob: bytes, source_name: str) -> RawAssetRecord:
+        """Persist the original bytes immutably and return a record. Raises on
+        empty input so empty content can never masquerade as a source asset."""
+        if not source_name.strip():
+            raise RawAssetStoreError("source_name is required")
+        if not blob:
+            raise RawAssetStoreError("empty original bytes cannot be stored")
+        digest = _sha256(blob)
+        dest = self._original_path(digest)
+        # Immutable write: only write when the content-addressed file is absent,
+        # and verify the hash after writing (no silent partial/corrupt writes).
+        if not dest.exists():
+            dest.write_bytes(blob)
+            if _sha256(dest.read_bytes()) != digest:
+                raise RawAssetStoreError("raw asset hash mismatch after write")
+        return RawAssetRecord(
+            sha256=digest,
+            size_bytes=len(blob),
+            source_name=source_name,
+            converted=None,
+        )
+
+    def _record_failure(self, digest: str, source_name: str, error: str) -> None:
+        payload = {
+            "sha256": digest,
+            "source_name": source_name,
+            "error": error,
+            "original_retained": True,
+        }
+        fp = self._failure_path(digest)
+        if not fp.exists():
+            fp.write_text(json.dumps(payload, ensure_ascii=True, indent=2), encoding="utf-8")
+
+
+def preserve_then_convert(
+    store: RawAssetStore,
+    blob: bytes,
+    source_name: str,
+    convert: Callable[[bytes], str],
+) -> RawAssetRecord:
+    """Persist the original first, then convert. On any converter failure the
+    original is retained and a durable failure record is written. Returns a
+    record whose `converted` is None and `error` populated on failure."""
+    original = store.store_original(blob, source_name)
+    try:
+        converted = convert(blob)
+    except BaseException as exc:  # noqa: BLE001 — we must not lose the original
+        store._record_failure(original.sha256, source_name, str(exc))
+        return RawAssetRecord(
+            sha256=original.sha256,
+            size_bytes=original.size_bytes,
+            source_name=source_name,
+            converted=None,
+            error=str(exc),
+        )
+    return RawAssetRecord(
+        sha256=original.sha256,
+        size_bytes=original.size_bytes,
+        source_name=source_name,
+        converted=converted,
+    )

app/release-manifest.json

@@ -24,9 +24,9 @@
   "dependency_lock": {
     "path": "uv.lock",
     "algorithm": "sha256",
-    "digest": "e103c5f9a46ca2e11d50460b610de648978b62e60f53bdd7dd97d81fcf121cf8",
+    "digest": "9916e6dba6d152cff045abf188218a0a3a533a786b3c3061738dc3a128aa4268",
     "format_version": 1,
-    "revision": 4
+    "revision": 5
   },
   "migrations": {
     "owners": [

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "sqlite-vec>=0.1.6",
     "loguru>=0.7",
     "structlog>=24.0",
-    "markitdown>=0.1",
+    "markitdown[pdf]>=0.1",
     "trafilatura>=1.6",
     "networkx>=3.0",
     "litellm==1.91.0",
@@ -61,7 +61,7 @@ ci = [
     "uvicorn[standard]>=0.22",
 ]
 ci-adapters = [
-    "markitdown>=0.1",
+    "markitdown[pdf]>=0.1",
     "newspaper4k>=0.9",
     "readabilipy>=0.3",
     "trafilatura>=1.6",

requirements.txt

@@ -11,7 +11,7 @@ apscheduler>=3.10
 sqlite-vec>=0.1.6
 loguru>=0.7
 structlog>=24.0
-markitdown>=0.1
+markitdown[pdf]>=0.1
 trafilatura>=1.6
 networkx>=3.0
 litellm==1.91.0

scripts/doctor_windows.ps1

@@ -0,0 +1,146 @@
+#requires -Version 7.0
+<#
+    SYNOPSIS
+      Windows/PowerShell 7 doctor for Cognitive-Loop-OS (AXW-007A).
+
+    DESCRIPTION
+      Detects the toolchain and Windows-environment prerequisites needed to run,
+      test, and package the project: Python, Node, Rust, PowerShell, Chinese and
+      space-containing paths, port availability, console encoding, and writable
+      directories.
+
+      Output is strictly sanitized: it never prints secrets, tokens, cookies,
+      credentials, private paths outside the declared scope, or personal body
+      text. Only names, versions, availability booleans and path-layout facts are
+      emitted. All results are returned as structured JSON on stdout; warnings go
+      to stderr.
+
+    OUTPUT
+      A single JSON object:
+      {
+        "schema_version": "axw.007a.v1",
+        "generated_at": "...",
+        "toolchain": { "python": {...}, "node": {...}, "rust": {...}, "powershell": {...} },
+        "paths": { "space_in_path": bool, "non_ascii_in_path": bool, "project_root": "<sanitized>", ... },
+        "ports": { "<label>": { "port": int, "available": bool } },
+        "encoding": { "console_codepage": int, "utf8_default": bool },
+        "writable": [ { "label": "...", "path": "...", "writable": bool } ],
+        "healthy": bool
+      }
+
+    PARAMETER ProjectRoot
+      Optional absolute path to the repository root. Defaults to the script's
+      parent (project checkout). Used only to detect path-layout facts and
+      writable-directory checks; the emitted path is a relative or sanitized form.
+
+    EXAMPLE
+      pwsh -NoProfile -File scripts/doctor_windows.ps1
+#>
+[CmdletBinding()]
+param(
+    [string]$ProjectRoot = (Resolve-Path (Join-Path $PSScriptRoot ".."))
+)
+
+$ErrorActionPreference = "Continue"
+
+function Test-CommandAvailable {
+    param([string]$Name)
+    return [bool](Get-Command $Name -ErrorAction SilentlyContinue)
+}
+
+function Get-Version([string]$Name) {
+    $cmd = Get-Command $Name -ErrorAction SilentlyContinue
+    if (-not $cmd) { return $null }
+    try {
+        $v = & $Name --version 2>$null | Select-Object -First 1
+        return $v
+    } catch { return $null }
+}
+
+function Test-PortAvailable([int]$Port) {
+    try {
+        $listener = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, $Port)
+        $listener.Start()
+        $listener.Stop()
+        return $true
+    } catch { return $false }
+}
+
+$result = [ordered]@{}
+
+# --- Toolchain -----------------------------------------------------------
+$result.schema_version = "axw.007a.v1"
+$result.generated_at = (Get-Date -Format o)
+
+$python = [ordered]@{ present = $false }
+if (Test-CommandAvailable "python") { $python.present = $true; $python.version = Get-Version "python" }
+if (Test-CommandAvailable "py")    { $python.launcher_present = $true }
+
+$node = [ordered]@{ present = $false }
+if (Test-CommandAvailable "node") { $node.present = $true; $node.version = Get-Version "node" }
+if (Test-CommandAvailable "npm")  { $node.npm_present = $true }
+
+$rust = [ordered]@{ present = $false }
+if (Test-CommandAvailable "cargo") { $rust.present = $true; $rust.version = Get-Version "cargo" }
+if (Test-CommandAvailable "rustc") { $rust.rustc_present = $true }
+
+$ps = [ordered]@{ present = $true; version = $PSVersionTable.PSVersion.ToString() }
+
+$result.toolchain = [ordered]@{
+    python      = $python
+    node        = $node
+    rust        = $rust
+    powershell  = $ps
+}
+
+# --- Path layout facts ---------------------------------------------------
+$project = [System.IO.Path]::GetFullPath((Resolve-Path $ProjectRoot).Path)
+$result.paths = [ordered]@{
+    space_in_path    = $project.Contains(" ")
+    non_ascii_in_path = ($project.ToCharArray() | Where-Object { [int]$_ -gt 127 } | Measure-Object).Count -gt 0
+    project_root_sanitized = (Split-Path $project -Leaf)   # leaf only; no full private path
+}
+
+# --- Port availability ---------------------------------------------------
+$result.ports = [ordered]@{}
+foreach ($port in @(8000, 8001, 9000, 4444, 9515)) {
+    $result.ports[$port.ToString()] = [ordered]@{ port = $port; available = (Test-PortAvailable $port) }
+}
+
+# --- Console encoding ----------------------------------------------------
+$result.encoding = [ordered]@{
+    console_codepage = [Console]::OutputEncoding.CodePage
+    utf8_default     = ([Console]::OutputEncoding.WebName -eq "utf-8")
+}
+
+# --- Writable directories (project-local only) ---------------------------
+$candidates = @(
+    @{ label = "project_root"; path = $project },
+    @{ label = "runtime_cache"; path = (Join-Path $project ".hermes/task-runtime") }
+)
+$result.writable = @()
+foreach ($c in $candidates) {
+    $target = $c.path
+    $writable = $false
+    try {
+        New-Item -ItemType Directory -Path $target -Force -ErrorAction Stop | Out-Null
+        $probe = Join-Path $target (".doctor_probe_" + [Guid]::NewGuid().ToString("N") + ".tmp")
+        Set-Content -Path $probe -Value "x" -ErrorAction Stop | Out-Null
+        Remove-Item -Path $probe -Force -ErrorAction Stop
+        $writable = $true
+    } catch {
+        $writable = $false
+    }
+    $result.writable += [ordered]@{
+        label     = $c.label
+        path      = (Split-Path $target -Leaf)   # sanitized leaf only
+        writable  = $writable
+    }
+}
+
+# --- Overall health ------------------------------------------------------
+$required = @("python")
+$missing = @($required | Where-Object { -not $result.toolchain.$_.present })
+$result.healthy = ($missing.Count -eq 0)
+
+$result | ConvertTo-Json -Depth 6