From 513d2aa2510a731f7e2ff923277a381dfc697d6a Mon Sep 17 00:00:00 2001 From: Jochen Hoenle <173445474+hoe-jo@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:26:13 +0200 Subject: [PATCH] [ai checker] refactor - refactor agent interface - refactor export generation - add rst report - move report generation from bazel build to bazel test - rework guidelines --- .bazelrc.ai_checker | 39 +- .../docs/user_guide/requirements.md | 6 +- .../rules_score/examples/seooc/design/BUILD | 14 + .../examples/seooc/design/design_context.md | 40 + validation/ai_checker/BUILD | 111 +- validation/ai_checker/DEVELOPMENT.md | 312 ++++++ validation/ai_checker/README.md | 264 +++-- .../ai_checker/_assets/class_diagram.puml | 126 ++- .../ai_checker/_assets/class_diagram.svg | 225 ++++- .../_assets/deployment_diagram.puml | 47 +- .../ai_checker/_assets/deployment_diagram.svg | 2 +- validation/ai_checker/ai_checker.bzl | 334 ++++--- .../architecture/architecture_guidelines.md | 42 + validation/ai_checker/guidelines/general.md | 78 +- .../project/score_architecture_levels.md | 23 + .../project/score_requirement_levels.md | 49 + .../requirements/requirements_guidelines.md | 80 ++ .../guidelines/requirements_guidelines.md | 69 -- validation/ai_checker/requirements.txt | 945 +++++++++++++----- validation/ai_checker/requirements.txt.in | 3 +- .../src/ai_checker/agents/__init__.py | 27 + .../agents}/_client_manager.py | 89 +- .../agents}/_errors.py | 0 .../agents}/_preflight.py | 11 +- .../src/ai_checker/agents/copilot_agent.py | 226 +++++ .../src/ai_checker/agents/langchain_agent.py | 69 ++ .../src/ai_checker/ai_checker_core.py | 104 +- .../src/ai_checker/analysis_agent.py | 114 +++ .../src/ai_checker/analysis_cache.py | 24 +- .../src/ai_checker/analysis_models.py | 2 +- .../extractors}/__init__.py | 7 +- .../extractors/architecture_extractor.py | 74 ++ .../base.py} | 21 + .../{ => extractors}/requirement_extractor.py | 97 +- .../src/ai_checker/guidelines_reader.py | 99 +- .../ai_checker/src/ai_checker/orchestrator.py | 439 ++++++-- .../src/ai_checker/reports/__init__.py | 13 + .../ai_checker/src/ai_checker/reports/base.py | 48 + .../src/ai_checker/reports/formatter.py | 152 +++ .../src/ai_checker/reports/html_renderer.py | 129 +++ .../src/ai_checker/reports/json_renderer.py | 29 + .../src/ai_checker/reports/metadata.py | 50 + .../src/ai_checker/reports/models.py | 54 + .../src/ai_checker/reports/report.html.j2 | 196 ++++ .../src/ai_checker/reports/rst_renderer.py | 122 +++ .../src/ai_checker/reports/text_utils.py | 102 ++ .../src/ai_checker/result_formatter.py | 607 ----------- .../src/copilot_adapter/_message_converter.py | 68 -- .../src/copilot_adapter/_tool_converter.py | 97 -- .../src/copilot_adapter/architecture.md | 206 ---- .../copilot_adapter/component_diagram.puml | 61 -- .../src/copilot_adapter/copilot_langchain.py | 389 ------- 52 files changed, 4120 insertions(+), 2415 deletions(-) create mode 100644 bazel/rules/rules_score/examples/seooc/design/design_context.md create mode 100644 validation/ai_checker/DEVELOPMENT.md create mode 100644 validation/ai_checker/guidelines/architecture/architecture_guidelines.md create mode 100644 validation/ai_checker/guidelines/project/score_architecture_levels.md create mode 100644 validation/ai_checker/guidelines/project/score_requirement_levels.md create mode 100644 validation/ai_checker/guidelines/requirements/requirements_guidelines.md delete mode 100644 validation/ai_checker/guidelines/requirements_guidelines.md create mode 100644 validation/ai_checker/src/ai_checker/agents/__init__.py rename validation/ai_checker/src/{copilot_adapter => ai_checker/agents}/_client_manager.py (75%) rename validation/ai_checker/src/{copilot_adapter => ai_checker/agents}/_errors.py (100%) rename validation/ai_checker/src/{copilot_adapter => ai_checker/agents}/_preflight.py (92%) create mode 100644 validation/ai_checker/src/ai_checker/agents/copilot_agent.py create mode 100644 validation/ai_checker/src/ai_checker/agents/langchain_agent.py create mode 100644 validation/ai_checker/src/ai_checker/analysis_agent.py rename validation/ai_checker/src/{copilot_adapter => ai_checker/extractors}/__init__.py (75%) create mode 100644 validation/ai_checker/src/ai_checker/extractors/architecture_extractor.py rename validation/ai_checker/src/ai_checker/{artefact_extractor.py => extractors/base.py} (67%) rename validation/ai_checker/src/ai_checker/{ => extractors}/requirement_extractor.py (76%) create mode 100644 validation/ai_checker/src/ai_checker/reports/__init__.py create mode 100644 validation/ai_checker/src/ai_checker/reports/base.py create mode 100644 validation/ai_checker/src/ai_checker/reports/formatter.py create mode 100644 validation/ai_checker/src/ai_checker/reports/html_renderer.py create mode 100644 validation/ai_checker/src/ai_checker/reports/json_renderer.py create mode 100644 validation/ai_checker/src/ai_checker/reports/metadata.py create mode 100644 validation/ai_checker/src/ai_checker/reports/models.py create mode 100644 validation/ai_checker/src/ai_checker/reports/report.html.j2 create mode 100644 validation/ai_checker/src/ai_checker/reports/rst_renderer.py create mode 100644 validation/ai_checker/src/ai_checker/reports/text_utils.py delete mode 100644 validation/ai_checker/src/ai_checker/result_formatter.py delete mode 100644 validation/ai_checker/src/copilot_adapter/_message_converter.py delete mode 100644 validation/ai_checker/src/copilot_adapter/_tool_converter.py delete mode 100644 validation/ai_checker/src/copilot_adapter/architecture.md delete mode 100644 validation/ai_checker/src/copilot_adapter/component_diagram.puml delete mode 100644 validation/ai_checker/src/copilot_adapter/copilot_langchain.py diff --git a/.bazelrc.ai_checker b/.bazelrc.ai_checker index 2274595c..a84ee444 100644 --- a/.bazelrc.ai_checker +++ b/.bazelrc.ai_checker @@ -12,27 +12,28 @@ # ******************************************************************************* ############################################################################### -## GitHub Copilot SDK - Environment (config:copilot) +## GitHub Copilot SDK - Environment ############################################################################### -# The Copilot CLI needs HOME (for stored OAuth credentials) and proxy vars -# (to reach api.github.com behind a corporate proxy). +# No environment configuration is required here. The AI analysis runs at TEST +# time and the AI test rules bake the required environment-variable inheritance +# (HOME, the GitHub tokens, and proxy variables) into each target via +# RunEnvironmentInfo. Running `bazel test //path/to:my_ai_check` inherits those +# variables from your shell automatically — no --config or --test_env needed. # -# These are scoped to --config=copilot so they don't affect other builds. -# The AI checker BUILD target applies this config automatically via a -# test --config=copilot line below. +# Provide credentials by exporting one of COPILOT_GITHUB_TOKEN / GH_TOKEN / +# GITHUB_TOKEN, or by logging in via the Copilot CLI (writes ~/.copilot/config.json). # # Auth docs: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md -# Auth -build:copilot --action_env=HOME -build:copilot --action_env=COPILOT_GITHUB_TOKEN -build:copilot --action_env=GH_TOKEN -build:copilot --action_env=GITHUB_TOKEN - -# Proxy (Node.js checks both upper and lowercase) -build:copilot --action_env=HTTP_PROXY -build:copilot --action_env=HTTPS_PROXY -build:copilot --action_env=NO_PROXY -build:copilot --action_env=http_proxy -build:copilot --action_env=https_proxy -build:copilot --action_env=no_proxy +############################################################################### +## Project-specific guidelines (optional) +############################################################################### +# General + element-type guidelines are built in. Project-specific details +# (e.g. requirement levels, architecture levels) are injected as graded rules +# via label flags, set once here instead of on every AI test target. +# +# Point them at your own filegroup of .md files, or reuse the bundled SCORE +# examples shown below. +# +# build --//validation/ai_checker:project_guidelines=//validation/ai_checker:score_project_guidelines +# build --//validation/ai_checker:project_architecture_guidelines=//validation/ai_checker:score_project_architecture_guidelines diff --git a/bazel/rules/rules_score/docs/user_guide/requirements.md b/bazel/rules/rules_score/docs/user_guide/requirements.md index fbce74b4..116044bf 100644 --- a/bazel/rules/rules_score/docs/user_guide/requirements.md +++ b/bazel/rules/rules_score/docs/user_guide/requirements.md @@ -199,7 +199,7 @@ The `tags = ["manual"]` attribute is strongly recommended to prevent the rule fr Run the check explicitly with: ```bash -bazel test //my/package:feature_requirements_ai_check --config=copilot +bazel test //my/package:feature_requirements_ai_check ``` | Attribute | Type | Required | Description | @@ -210,7 +210,9 @@ bazel test //my/package:feature_requirements_ai_check --config=copilot | `score_threshold` | string | no | Minimum average quality score from 0 to 10 to pass the test (default: `"0.0"`) | | `guidelines` | label | no | Filegroup of guideline Markdown files to override the built-in guidelines | -**Output files** (written to `bazel-bin/`): +**Output files** (the AI analysis runs at test time; reports are written to the +test's undeclared-outputs archive at +`bazel-testlogs///test.outputs/outputs.zip`): | File | Content | |---|---| diff --git a/bazel/rules/rules_score/examples/seooc/design/BUILD b/bazel/rules/rules_score/examples/seooc/design/BUILD index 90755164..44195f4e 100644 --- a/bazel/rules/rules_score/examples/seooc/design/BUILD +++ b/bazel/rules/rules_score/examples/seooc/design/BUILD @@ -14,6 +14,7 @@ load( "//bazel/rules/rules_score:rules_score.bzl", "architectural_design", ) +load("//validation/ai_checker:ai_checker.bzl", "architecture_ai_test") architectural_design( name = "sample_seooc_design", @@ -29,3 +30,16 @@ architectural_design( ], visibility = ["//visibility:public"], ) + +# Background context (read-only) for the AI architecture review. +filegroup( + name = "design_context", + srcs = ["design_context.md"], +) + +architecture_ai_test( + name = "sample_seooc_design_ai_test", + context = ":design_context", + designs = [":sample_seooc_design"], + tags = ["manual"], +) diff --git a/bazel/rules/rules_score/examples/seooc/design/design_context.md b/bazel/rules/rules_score/examples/seooc/design/design_context.md new file mode 100644 index 00000000..a791721d --- /dev/null +++ b/bazel/rules/rules_score/examples/seooc/design/design_context.md @@ -0,0 +1,40 @@ + + +# Design Context: Sample SEooC + +Background material provided to the AI reviewer as read-only reference. It is +**not** graded; it only helps the reviewer interpret the architecture under +review. + +## Scope + +This is a Safety Element out of Context (SEooC). The component is developed +without a concrete vehicle-level item, so assumptions of use (AoU) stand in for +the missing system context. + +## Components + +- The static design (`static_design.puml`) describes the component structure and + its public interfaces. +- The dynamic design (`dynamic_design.puml`) describes the runtime interaction + between the component and its environment. +- The public API (`public_api.puml`) defines the interfaces exposed to + integrators. + +## Assumptions of Use + +- Integrators are responsible for satisfying the documented assumptions of use + before relying on the component's safety claims. +- The execution environment provides the resources declared in the static + design. diff --git a/validation/ai_checker/BUILD b/validation/ai_checker/BUILD index ffe487bc..3ff90045 100644 --- a/validation/ai_checker/BUILD +++ b/validation/ai_checker/BUILD @@ -28,47 +28,127 @@ filegroup( # Default requirements engineering guidelines filegroup( name = "default_guidelines", - srcs = glob(["guidelines/*.md"]), + srcs = ["guidelines/general.md"] + glob(["guidelines/requirements/*.md"]), visibility = ["//visibility:public"], ) -# Core AI checker library (analysis framework) +# Default architecture design guidelines +filegroup( + name = "default_architecture_guidelines", + srcs = ["guidelines/general.md"] + glob(["guidelines/architecture/*.md"]), + visibility = ["//visibility:public"], +) + +# Empty default for the project-guideline flags below. Consumer repos override +# the flags in their .bazelrc to inject project-specific guidelines once, +# instead of setting an attribute on every AI test target. +filegroup( + name = "empty_guidelines", + srcs = [], + visibility = ["//visibility:public"], +) + +# Project-specific requirement guidelines (graded). Override once via: +# build --//validation/ai_checker:project_guidelines=//path/to:my_guidelines +label_flag( + name = "project_guidelines", + build_setting_default = ":empty_guidelines", + visibility = ["//visibility:public"], +) + +# Project-specific architecture guidelines (graded). Override once via: +# build --//validation/ai_checker:project_architecture_guidelines=//path/to:my_guidelines +label_flag( + name = "project_architecture_guidelines", + build_setting_default = ":empty_guidelines", + visibility = ["//visibility:public"], +) + +# Example SCORE project guidelines, referenced by the flags above when a repo +# opts into the SCORE process levels. +filegroup( + name = "score_project_guidelines", + srcs = ["guidelines/project/score_requirement_levels.md"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "score_project_architecture_guidelines", + srcs = ["guidelines/project/score_architecture_levels.md"], + visibility = ["//visibility:public"], +) + +# Core AI checker library (analysis framework). +# Deliberately free of any AI-SDK / LangChain dependency: it depends only on +# the AnalysisAgent interface, which concrete agents implement. +# Core analysis framework + extractors + the AnalysisAgent interface. +# No AI-SDK / LangChain dependency. The agents/ subpackage (concrete backends) +# is intentionally excluded — those are separate libraries below. py_library( name = "ai_checker_core", - srcs = glob(["src/ai_checker/*.py"]), + srcs = glob([ + "src/ai_checker/*.py", + "src/ai_checker/extractors/*.py", + "src/ai_checker/reports/*.py", + ]), + # Jinja2 report templates are loaded at runtime relative to the package, + # so they must travel in the library's runfiles. + data = glob(["src/ai_checker/reports/*.j2"]), imports = ["src"], visibility = ["//visibility:public"], deps = [ "@trlc//trlc", requirement("bigtree"), + requirement("jinja2"), + requirement("markupsafe"), requirement("pydantic"), requirement("pydot"), requirement("pyyaml"), ], ) -# LangChain adapter for GitHub Copilot SDK +# Default AI backend: GitHub Copilot SDK agent (no LangChain). py_library( - name = "copilot_langchain", + name = "copilot_agent", srcs = [ - "src/copilot_adapter/__init__.py", - "src/copilot_adapter/_client_manager.py", - "src/copilot_adapter/_errors.py", - "src/copilot_adapter/_message_converter.py", - "src/copilot_adapter/_preflight.py", - "src/copilot_adapter/_tool_converter.py", - "src/copilot_adapter/copilot_langchain.py", + "src/ai_checker/agents/__init__.py", + "src/ai_checker/agents/_client_manager.py", + "src/ai_checker/agents/_errors.py", + "src/ai_checker/agents/_preflight.py", + "src/ai_checker/agents/copilot_agent.py", ], imports = ["src"], visibility = ["//visibility:public"], deps = [ - requirement("langchain-core"), + ":ai_checker_core", requirement("github-copilot-sdk"), requirement("pydantic"), ], ) -# Default orchestrator (uses GitHub Copilot SDK as default AI backend) +# Optional LangChain adapter: wraps any LangChain BaseChatModel (e.g. ChatOpenAI) +# as an AnalysisAgent. Used only when a custom langchain model is supplied via a +# custom ai_model target's create_agent(). +py_library( + name = "langchain_agent", + srcs = [ + "src/ai_checker/agents/__init__.py", + "src/ai_checker/agents/_errors.py", + "src/ai_checker/agents/langchain_agent.py", + ], + imports = ["src"], + visibility = ["//visibility:public"], + deps = [ + ":ai_checker_core", + requirement("langchain-core"), + ], +) + +# Default orchestrator (uses the Copilot SDK agent as default AI backend). +# Intentionally does NOT depend on :langchain_agent — the default path is +# LangChain-free. A consumer wanting the LangChain path supplies a custom +# ai_model target whose create_agent() returns a LangChainAgent and which +# depends on :langchain_agent itself. py_binary( name = "orchestrator", srcs = ["src/ai_checker/orchestrator.py"], @@ -77,8 +157,7 @@ py_binary( visibility = ["//visibility:public"], deps = [ ":ai_checker_core", - ":copilot_langchain", - requirement("langchain-core"), + ":copilot_agent", ], ) diff --git a/validation/ai_checker/DEVELOPMENT.md b/validation/ai_checker/DEVELOPMENT.md new file mode 100644 index 00000000..3a19853b --- /dev/null +++ b/validation/ai_checker/DEVELOPMENT.md @@ -0,0 +1,312 @@ + + +# AI Checker — Development Guide + +Technical reference for developing and extending the AI Checker. For usage and +Bazel integration, see [README.md](README.md). + +## Architecture + +The AI Checker is a single `ai_checker` package under the `src/` import root, +clustered by responsibility: + +| Path | Purpose | +|------|---------| +| `src/ai_checker/` | Core analysis framework (orchestrator, scoring, caching, reporting, guidelines reader) + the `AnalysisAgent` interface. **No AI-SDK / LangChain dependency.** | +| `src/ai_checker/extractors/` | `ArtefactExtractor` implementations: `base.py` (ABC), `requirement_extractor.py` (TRLC), `architecture_extractor.py` (raw PlantUML). | +| `src/ai_checker/agents/` | AI backends: `CopilotAgent` (default, direct Copilot SDK) and the optional `LangChainAgent` (wraps any LangChain `BaseChatModel`), plus Copilot session plumbing (`_client_manager`, `_preflight`, `_errors`). | + +The core never talks to an SDK directly; it depends on the `AnalysisAgent` +interface (`analyze(system_prompt, artefacts_text) -> AnalysisResults`). +`CopilotAgent` is the default implementation; `LangChainAgent` wraps any +LangChain `BaseChatModel` for custom backends. + +### Diagrams + +**Deployment overview:** + +![Deployment Diagram](_assets/deployment_diagram.svg) + +**Class relationships:** + +![Class Diagram](_assets/class_diagram.svg) + +## Key Components + +### `AIChecker` (`src/ai_checker/ai_checker_core.py`) + +Performs the async AI analysis. Responsibilities: + +- Splits artefacts into batches (by count via `--batch-size` and by total + character length via `--max-batch-chars`) +- Processes batches concurrently, rate-limited by an `asyncio.Semaphore` +- Calls `AnalysisAgent.analyze(system_prompt, artefacts_text)` per batch +- Manages the optional result cache (`AnalysisCache`) + +### `AnalysisAgent` (`src/ai_checker/analysis_agent.py`) + +The interface between the core and any AI backend. One async method, +`analyze(system_prompt, artefacts_text) -> AnalysisResults`, plus a +`get_usage() -> Usage` accessor (backends call the protected `_record_usage()` +to accumulate tokens / cost / AI credits) and an `aclose()` hook for releasing +resources. Keeps the core free of any SDK / LangChain dependency. + +### `AnalysisOrchestrator` (`src/ai_checker/orchestrator.py`) + +Top-level coordinator. Selects the extractor, builds the agent, assembles the +system prompt, runs the AI checker, and exposes the CLI entry point (`main()`). + +The system prompt is layered in this order: + +1. **General + type guidelines** — concatenated from the `--guidelines` + directory (e.g. `general.md` + the element-type guideline). +2. **Project-specific guidelines** — graded rules from `--project-guidelines`, + appended under a `# PROJECT-SPECIFIC GUIDELINES` heading. +3. **Background context** — read-only reference material from `--context-file`, + appended under a clearly labelled "reference only — not graded" heading. + +### `GuidelinesReader` (`src/ai_checker/guidelines_reader.py`) + +Reads text documents into a `name -> content` mapping, from either a directory +(guidelines, `*.md`) or an explicit file list (project guidelines / background +context, `.md` / `.puml`), filtered by extension. + +### `RequirementExtractor` / `ArchitectureExtractor` (`src/ai_checker/extractors/`) + +`ArtefactExtractor` implementations selected by `--artefact-type`. +`RequirementExtractor` parses TRLC files via the TRLC Python API (only objects +under `--input` are analyzed; `--deps` are loaded for link resolution). +`ArchitectureExtractor` reads the raw `.puml` source of architecture diagrams. +Both return `dict[str, dict[str, Any]]`. + +### Reports (`src/ai_checker/reports/`) + +`ResultFormatter` (`reports/formatter.py`) builds **one** `AnalysisReport` +(`reports/models.py`) in memory — report metadata, guideline texts, and the +per-artefact analyses — then renders the requested format via a +`ReportRenderer` (`reports/base.py`), chosen by output extension: + +| Extension | Renderer | Notes | +|-----------|----------|-------| +| `.json` (default) | `JsonRenderer` | Self-contained envelope; keeps top-level `analyses`. | +| `.html` | `HtmlRenderer` | Styled page + per-guideline `.md` subpages. | +| `.rst` | `RstRenderer` | Standalone reStructuredText + per-guideline `.rst` subpages. | + +Shared helpers live in `reports/text_utils.py` (slugs, severity, markdown→HTML) +and `reports/metadata.py` (git hash, timestamp). Add a format by subclassing +`ReportRenderer` and registering it in `reports/formatter.py`. + +## Agents + +The `ai_checker.agents` package provides AI backends. The core depends only on +the `AnalysisAgent` interface; this package supplies two implementations: + +| Class | Path | Role | +|---|---|---| +| **`CopilotAgent`** | `copilot_agent.py` | **Default.** Talks to the **GitHub Copilot SDK** directly — no LangChain. Owns one CLI session per request, embeds the JSON schema in the system prompt, parses the reply into `AnalysisResults`. | +| `LangChainAgent` | `langchain_agent.py` | Optional. Wraps any LangChain `BaseChatModel` (e.g. `ChatOpenAI`) as an `AnalysisAgent`, via `with_structured_output(AnalysisResults)`. Used when a custom model is supplied. | + +The two backends are independent: `CopilotAgent` needs only the Copilot SDK, +`LangChainAgent` needs only `langchain_core`. The Copilot session plumbing +(`_client_manager`, `_preflight`, `_errors`) is used by `CopilotAgent` only. + +### Module Responsibilities + +#### `copilot_agent.py` — `CopilotAgent` (default) + +Implements `AnalysisAgent.analyze(system_prompt, artefacts_text) -> AnalysisResults` +directly against the Copilot SDK: + +1. `CopilotClientManager.ensure_client()` (shared pre-flight + session plumbing). +2. Build the system message = `system_prompt` + a JSON-schema instruction for + `AnalysisResults`; send `artefacts_text` as the prompt. +3. Parse the model's text reply into `AnalysisResults` (`_extract_json_object` + scans for the first balanced `{...}` object, tolerating fences/prose, then + validates against the schema). +4. Usage capture: subscribe to the session's `assistant.usage` events and + accumulate the typed `AssistantUsageData` fields (input/output tokens, + experimental `cost`, and `copilotUsage.totalNanoAiu` for AI credits) into a + `Usage`, recorded via `_record_usage()`. + +#### `langchain_agent.py` — `LangChainAgent` (optional) + +Adapts any LangChain `BaseChatModel` to `AnalysisAgent` via +`model.with_structured_output(AnalysisResults).ainvoke([SystemMessage, HumanMessage])`. +Only used when a custom `create_agent()` returns one, e.g.: + +```python +from ai_checker.agents.langchain_agent import LangChainAgent +from langchain_openai import ChatOpenAI + +def create_agent(model_name): + return LangChainAgent(ChatOpenAI(model=model_name)) +``` + +#### `_client_manager.py` — `CopilotClientManager` + +Owns the lifecycle of the single `CopilotClient` / CLI subprocess. The same +subprocess is reused across calls. Pre-flight sequence executed once before the +first request: + +1. Resolve the `copilot_cli` binary path (Bazel `copy_executables` workaround). +2. Verify the binary exists and is executable. +3. Hard-fail if no auth source is found (`COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, + `GITHUB_TOKEN`, or `~/.copilot/config.json`). +4. Warn (non-fatal) about missing `$HOME` or `HTTPS_PROXY`. +5. Spawn the subprocess and authenticate via `get_auth_status()`. + +#### `_preflight.py` + +Stateless helpers called by `CopilotClientManager` before startup: +`resolve_copilot_cli_path()`, `check_cli_binary()`, `check_auth_sources()`, +`check_environment()`, `describe_auth_sources()`. + +#### `_errors.py` + +- `CopilotSetupError` — a `RuntimeError` subclass raised for any configuration + or startup failure. Carries an actionable message for the user. +- `AUTH_ENV_VARS` — ordered list of accepted auth environment variables. + +### Why JSON-in-prompt instead of structured output / tool calling? + +The GitHub Copilot SDK (v0.3.0) does **not** expose any native +structured-output mechanism: `SessionConfig` has no `response_format`, +`output_schema`, or `json_schema` field, and `send_and_wait` returns plain text +in `response.data.content`. + +The SDK does expose a `tools` (function-calling) mechanism, but it provides **no +benefit** for forcing a structured result here: + +- The SDK's `Tool` is `{name, description, handler, parameters}` and nothing in + the SDK constrains the generated tool arguments to the `parameters` schema, so + any tool-call payload would still have to be validated manually — exactly like + the text reply today. +- The Copilot CLI's Claude model (reasoning enabled) ignores tool-calling + instructions and always replies in plain text, so registering a tool yields no + tool calls. +- Tool calling is designed for *agentic actions* (fetching data, running code), + not for output formatting; we only need a single structured result. + +Therefore `CopilotAgent` embeds the `AnalysisResults` JSON schema in the system +prompt and parses the reply (`_extract_json_object` + `model_validate`). The +`LangChainAgent` path achieves structured output natively via the wrapped +model's `with_structured_output(AnalysisResults)`. + +### Authentication + +The Copilot CLI requires a valid GitHub OAuth token. `_preflight.py` checks the +following sources in priority order during pre-flight: + +| Priority | Source | Notes | +|---|---|---| +| 1 | `COPILOT_GITHUB_TOKEN` env var | Recommended for explicit Copilot usage | +| 2 | `GH_TOKEN` env var | GitHub CLI compatible | +| 3 | `GITHUB_TOKEN` env var | GitHub Actions compatible | +| 4 | `~/.copilot/config.json` | Written by `gh copilot` interactive login | + +If **none** is present, `CopilotClientManager` raises a `CopilotSetupError` +before spawning the subprocess. If at least one exists, the CLI is started and +`get_auth_status()` confirms the token is accepted. + +In the Bazel setup the analysis runs at **test time** (see +[Bazel Test Rules](#bazel-test-rules)). The test rules bake the required +environment-variable inheritance into each target via `RunEnvironmentInfo` +(`inherited_environment`), so `HOME`, the GitHub tokens, and the proxy variables +are inherited from the invoking shell without any `--config=copilot` / +`--test_env` flag. `HOME` is essential because the test runner otherwise resets +it to `$TEST_TMPDIR`, hiding `~/.copilot/config.json`. + +### Error Handling Summary + +| Failure point | Exception type | What is logged | +|---|---|---| +| CLI binary missing / not executable | `CopilotSetupError` | Path checked, alternatives suggested | +| No auth source found | `CopilotSetupError` | Lists all env vars and config file path | +| Copilot SDK startup error | `CopilotSetupError` | Wraps original exception + auth description | +| Model returns no JSON object | `ValueError` | Full LLM output | +| Model returns malformed JSON | `ValueError` | `json.JSONDecodeError` position + full LLM output | +| Model returns wrong JSON structure | `ValueError` | Pydantic field-level `ValidationError` + full LLM output | + +## Caching Design + +`AnalysisCache` keys results by `SHA-256(artefacts_json + guidelines + model_name)`. +It is **only** usable via the CLI `--cache` flag. The Bazel test rules +deliberately omit `--cache`; the AI tests are tagged `external` (their result is +non-deterministic), so Bazel never caches them and a stale cache cannot mask a +real regression. + +## Bazel Test Rules + +The `trlc_requirements_ai_test` / `architecture_ai_test` macros wrap private +rules (`_trlc_requirements_ai_test` / `_architecture_ai_test`) that run the AI +analysis **at test time**, not as a build action. Rationale: + +- The AI call is inherently non-hermetic (outbound network + user credentials). + Tests are the idiomatic home for non-hermetic work, and environment + inheritance is baked into the target via `RunEnvironmentInfo` — cleaner than + `--action_env` on a build action and requires no `--config` flag. +- Reports are written to `$TEST_UNDECLARED_OUTPUTS_DIR`, so they are captured in + `bazel-testlogs/.../test.outputs/outputs.zip` instead of `bazel-bin`. + +Flow: + +1. `_run_ai_analysis` (`ai_checker.bzl`) bakes the orchestrator arguments + (runfiles-relative `short_path`s) and writes a small launcher inline via + `ctx.actions.write`. Guideline files are passed individually with + `--guidelines-file` (the default guideline sets span several directories, so + a single derived directory would drop some); the requirement grading scope + is defined by the explicit `--req-file`s rather than a derived directory. +2. Per the [Bazel Test Encyclopedia](https://bazel.build/reference/test-encyclopedia), + a test starts with its working directory at `$TEST_SRCDIR/$TEST_WORKSPACE` + (the runfiles root), so the launcher needs no runfiles probing: it simply + `exec`s the orchestrator `py_binary` with the baked arguments and + `--score-threshold`. +3. The orchestrator detects `$TEST_UNDECLARED_OUTPUTS_DIR` and writes all of its + reports (`analysis.json` / `.html` / `.rst`, the guideline pages and + `debug.log`) there itself, so the launcher carries no output-path plumbing. +4. The orchestrator runs the analysis and, when `--score-threshold` is set, + computes the average score and exits non-zero if it is below the threshold — + so a failing score fails the test directly (no separate checker script). + +The macros inject default tags (`no-sandbox`, `requires-network`, `external`) +and merge any caller-supplied `tags`. Environment inheritance is provided by +`RunEnvironmentInfo(inherited_environment = [...])`. + +## Adding a New Artefact Type + +1. Subclass `ArtefactExtractor` (`src/ai_checker/extractors/base.py`) and + implement `extract() -> dict[str, dict[str, Any]]`. +2. Add a new `--artefact-type` value and select your extractor in + `AnalysisOrchestrator.analyze_directory()`. +3. Add a corresponding Bazel rule in `ai_checker.bzl` following the pattern of + `_trlc_requirements_ai_test_impl` / `_architecture_ai_test_impl`. + +## Adding a Custom AI Backend + +Provide a `create_agent(model_name) -> AnalysisAgent` factory in an +`ai_model.py` file and wire it via the `_custom_ai_model` attribute (see +[README.md](README.md)). The agent must implement: + +```python +async def analyze(self, system_prompt: str, artefacts_text: str) -> AnalysisResults +``` + +To reuse a LangChain model, return the bundled `LangChainAgent` wrapper. + +## Updating Python Dependencies + +```bash +# Core + Copilot SDK dependencies +bazel run //validation/ai_checker:requirements.update +``` diff --git a/validation/ai_checker/README.md b/validation/ai_checker/README.md index 40c616a1..64fdcbf6 100644 --- a/validation/ai_checker/README.md +++ b/validation/ai_checker/README.md @@ -29,7 +29,7 @@ produces: - a list of **suggestions** for improvement - a numerical **quality score** from 0 to 10 -Results are written as a JSON file and, optionally, an HTML report. +Results are written as a JSON envelope plus HTML and reStructuredText reports. ### Prerequisites @@ -39,7 +39,9 @@ Results are written as a JSON file and, optionally, an HTML report. ### Running a Check -Add a rule to your `BUILD` file and run it with `--config=copilot`: +Add a rule to your `BUILD` file and run it like any other test — the rule bakes +the required environment-variable inheritance (credentials + proxy) into the +target, so no extra `--config` or `--test_env` flag is needed: ```starlark load("@score_tooling//validation/ai_checker:ai_checker.bzl", "trlc_requirements_ai_test") @@ -53,7 +55,7 @@ trlc_requirements_ai_test( ``` ```bash -bazel test //path/to:requirements_ai_check --config=copilot +bazel test //path/to:requirements_ai_check ``` The `tags = ["manual"]` attribute is recommended to prevent the rule from @@ -70,7 +72,7 @@ guidelines. trlc_requirements_ai_test( name = "requirements_ai_check", reqs = [":my_requirements"], # required: targets providing TrlcProviderInfo - model = "anthropic/claude-sonnet-4-5", # optional: AI model to use + model = "claude-sonnet-4.6", # optional: AI model to use score_threshold = "6.0", # optional: minimum average score to pass (0–10) guidelines = "//my/org:guidelines", # optional: override default guideline filegroup tags = ["manual"], @@ -80,9 +82,10 @@ trlc_requirements_ai_test( | Attribute | Description | Required | Default | |-----------|-------------|----------|---------| | `reqs` | Label list of targets providing `TrlcProviderInfo` | Yes | — | -| `model` | AI model identifier | No | `"anthropic/claude-sonnet-4-5"` | +| `model` | AI model identifier | No | `"claude-sonnet-4.6"` | | `score_threshold` | Minimum average score (0–10) to pass the test | No | `"0.0"` | | `guidelines` | Filegroup of guideline markdown files | No | `default_guidelines` | +| `context` | Filegroup of background-context files (`.md` / `.puml`) injected as read-only reference material | No | — | #### `architecture_ai_test` @@ -93,7 +96,7 @@ guidelines. architecture_ai_test( name = "architecture_ai_check", designs = [":my_architectural_design"], # required: targets providing ArchitecturalDesignInfo - model = "anthropic/claude-sonnet-4-5", + model = "claude-sonnet-4.6", score_threshold = "6.0", tags = ["manual"], ) @@ -102,36 +105,57 @@ architecture_ai_test( | Attribute | Description | Required | Default | |-----------|-------------|----------|---------| | `designs` | Label list of targets providing `ArchitecturalDesignInfo` | Yes | — | -| `model` | AI model identifier | No | `"anthropic/claude-sonnet-4-5"` | +| `model` | AI model identifier | No | `"claude-sonnet-4.6"` | | `score_threshold` | Minimum average score (0–10) to pass the test | No | `"0.0"` | | `guidelines` | Filegroup of guideline markdown files | No | `default_architecture_guidelines` | +| `context` | Filegroup of background-context files (`.md` / `.puml`) injected as read-only reference material | No | — | + +> Architecture review reads the **raw PlantUML source** of the design's +> diagrams (not the parsed FlatBuffers binaries). ### Output -Each test rule produces two output files: +The AI analysis runs **at test time** (the test action launches the analysis), +so the reports are written to the test's undeclared-outputs directory and packed +into the test log archive. Each test produces three report files (one set per +test, so the names are fixed rather than prefixed): | File | Content | |------|---------| -| `_analysis.json` | Machine-readable results (scores, findings, suggestions) | -| `_analysis.html` | Interactive HTML report | +| `analysis.json` | Self-contained report envelope: `metadata`, `guidelines`, `analyses` (scores, findings, suggestions) | +| `analysis.html` | Interactive HTML report | +| `analysis.rst` | Standalone reStructuredText report | The HTML report shows a color-coded score card per artefact, linked guideline -reference pages, and summary statistics. Both files land in `bazel-bin/`. +reference pages, and summary statistics. The JSON is a self-contained envelope +(model/timestamp/git metadata + guideline texts + per-artefact analyses), so it +fully captures the report. + +Retrieve the reports after a test run from the undeclared-outputs archive: + +```bash +bazel test //path/to:requirements_ai_check +unzip -o bazel-testlogs/path/to/requirements_ai_check/test.outputs/outputs.zip -d /tmp/ai_report +``` ### Debug Output -To inspect the raw prompt sent to the AI model: +A verbose debug log (`debug.log`) is always written alongside the reports +in the same undeclared-outputs archive. It contains the raw prompt sent to the +AI model and response timing. Extract it the same way: ```bash -bazel test //path/to:requirements_ai_check --config=copilot --output_groups=debug -cat bazel-bin/path/to/requirements_ai_check_debug.log +bazel test //path/to:requirements_ai_check +unzip -p bazel-testlogs/path/to/requirements_ai_check/test.outputs/outputs.zip \ + debug.log ``` (custom-ai-model)= ### Custom AI Model -To use a provider other than GitHub Copilot, point `_custom_ai_model` at a -`py_binary` or `py_library` target that exposes a `create_chat_model()` function: +To use a provider other than the default Copilot SDK agent, point +`_custom_ai_model` at a `py_binary` or `py_library` target that exposes a +`create_agent()` function returning an `AnalysisAgent`: ```starlark trlc_requirements_ai_test( @@ -141,8 +165,8 @@ trlc_requirements_ai_test( ) ``` -See the [Integration Guide](#integration-guide) for details on implementing a -[Integration Guide](#integration-guide) for full details. +See the [Integration Guide](#integration-guide) for full details on implementing +a custom agent. --- @@ -153,40 +177,29 @@ This section describes how to use the AI Checker from another Bazel repository (e.g., a consumer workspace that references this repo via a Bazel registry or `git_repository`). -### Step 1 — Import the Bazel Config +### Step 1 — Provide Credentials -Add this line to your root `.bazelrc` to pull in the Copilot environment -configuration: +The AI analysis runs at **test time**, and the test rules bake the required +environment-variable inheritance into each target via `RunEnvironmentInfo`. When +you run `bazel test`, the test inherits `HOME`, the GitHub tokens, and the proxy +variables from your shell automatically — there is **no** `--config=copilot` or +`--test_env` flag to set, and nothing to copy into your root `.bazelrc`. -```text -try-import %workspace%/.bazelrc.ai_checker -``` +> `HOME` matters because the test runner otherwise resets it to a private temp +> directory, which would hide the Copilot CLI's `~/.copilot/config.json`. -Copy `.bazelrc.ai_checker` from this repository into your workspace root. -It forwards the authentication and proxy variables the Copilot CLI needs -into Bazel's sandbox: +Just make sure one credential source is present in your shell before running the +test (see the table below). + +Optionally, import the bundled `.bazelrc.ai_checker` to enable the +project-specific guideline flags (it contains **no** environment configuration): ```text -build:copilot --action_env=HOME -build:copilot --action_env=COPILOT_GITHUB_TOKEN -build:copilot --action_env=GH_TOKEN -build:copilot --action_env=GITHUB_TOKEN -build:copilot --action_env=HTTP_PROXY -build:copilot --action_env=HTTPS_PROXY -build:copilot --action_env=NO_PROXY -build:copilot --action_env=http_proxy -build:copilot --action_env=https_proxy -build:copilot --action_env=no_proxy +try-import %workspace%/.bazelrc.ai_checker ``` -**Why `--config=copilot`?** -Bazel sandboxes strip the host environment by default. The Copilot SDK's -Node.js CLI needs `HOME` (for stored OAuth tokens) and proxy variables (to -reach `api.github.com`) to be explicitly forwarded. These are scoped to -`config:copilot` so they do not affect other build actions. - -**Authentication** — at least one of the following must be available inside -the sandbox: +**Authentication** — at least one of the following must be available in your +environment: | Variable | Purpose | |----------|---------| @@ -206,7 +219,7 @@ load("@score_tooling//validation/ai_checker:ai_checker.bzl", trlc_requirements_ai_test( name = "requirements_ai_check", reqs = [":my_requirements"], # target providing TrlcProviderInfo - model = "anthropic/claude-sonnet-4-5", + model = "claude-sonnet-4.6", score_threshold = "6.0", # fail if average score < 6.0 tags = ["manual"], # recommended: exclude from //... ) @@ -215,7 +228,7 @@ trlc_requirements_ai_test( architecture_ai_test( name = "architecture_ai_check", designs = [":my_architectural_design"], # target providing ArchitecturalDesignInfo - model = "anthropic/claude-sonnet-4-5", + model = "claude-sonnet-4.6", score_threshold = "6.0", tags = ["manual"], ) @@ -226,20 +239,29 @@ AI analysis runs during routine `bazel test //...` sweeps. Run AI tests by targeting them explicitly: ```bash -bazel test //path/to:requirements_ai_check --config=copilot +bazel test //path/to:requirements_ai_check ``` | Attribute | Description | Required | Default | |-----------|-------------|----------|---------| | `reqs` / `designs` | Targets providing `TrlcProviderInfo` or `ArchitecturalDesignInfo` | Yes | — | -| `model` | AI model identifier | No | `"anthropic/claude-sonnet-4-5"` | +| `model` | AI model identifier | No | `"claude-sonnet-4.6"` | | `score_threshold` | Minimum average score (0–10) to pass | No | `"0.0"` | | `guidelines` | Custom guideline filegroup | No | `default_guidelines` / `default_architecture_guidelines` | +| `context` | Background-context filegroup (`.md` / `.puml`), read-only reference material | No | — | + +### Guidelines -### Overriding Guidelines +Guidelines are layered, so projects only supply what is specific to them: -Each rule uses a default `guidelines` filegroup. Override per target to -supply organisation-specific rules: +| Layer | Scope | Source | +|-------|-------|--------| +| **General** | Review methodology, scoring, result format — applies to every element type | `guidelines/general.md` | +| **Type** | Generic rules for one element type (requirements *or* architecture) | `guidelines/requirements/` · `guidelines/architecture/` | +| **Project** | Project-specific details (e.g. requirement levels, architecture levels) | Set via a flag — see below | + +The general and type layers are built in. To override them per target, set the +`guidelines` attribute to your own filegroup: ```starlark trlc_requirements_ai_test( @@ -249,6 +271,22 @@ trlc_requirements_ai_test( ) ``` +### Project-Specific Guidelines (set once) + +Project details are injected as **graded** rules via label flags, so you set +them once in `.bazelrc` instead of on every target: + +```text +build --//validation/ai_checker:project_guidelines=//my/org:my_req_guidelines +build --//validation/ai_checker:project_architecture_guidelines=//my/org:my_arch_guidelines +``` + +Each flag points at a `filegroup` of `.md` files. Bundled SCORE examples are +available as `//validation/ai_checker:score_project_guidelines` and +`//validation/ai_checker:score_project_architecture_guidelines`. When unset, no +project guidelines are added. + + ### Custom AI Model (Bazel) To substitute a different AI backend at the Bazel level, provide a @@ -262,15 +300,31 @@ trlc_requirements_ai_test( ) ``` -The file must expose `create_chat_model(model_name, max_completion_tokens)`. +The file must expose `create_agent(model_name) -> AnalysisAgent`. The agent +implements a single async method: + +```python +async def analyze(self, system_prompt: str, artefacts_text: str) -> AnalysisResults +``` + +To reuse a LangChain model, return the bundled `LangChainAgent` wrapper: + +```python +from ai_checker.agents.langchain_agent import LangChainAgent + +def create_agent(model_name): + return LangChainAgent(MyLangChainChatModel(model=model_name)) +``` ### Debug Output -To inspect the raw input sent to the AI model and response timing: +To inspect the raw input sent to the AI model and response timing, extract the +always-on debug log from the test's undeclared-outputs archive: ```bash -bazel build //path/to:requirements_ai_check --config=copilot --output_groups=debug -cat bazel-bin/path/to/requirements_ai_check_debug.log +bazel test //path/to:requirements_ai_check +unzip -p bazel-testlogs/path/to/requirements_ai_check/test.outputs/outputs.zip \ + debug.log ``` The debug log contains: @@ -283,99 +337,5 @@ The debug log contains: ## Developer Guide -### Architecture - -The AI Checker is organized into two source layers and one extension point: - -| Directory | Purpose | -|-----------|---------| -| `src/ai_checker/` | Core analysis framework (extraction, scoring, caching, reporting). Depends on `langchain-core` for the `BaseChatModel` interface. | -| `src/copilot_adapter/` | `ChatCopilot` — LangChain `BaseChatModel` wrapper for the GitHub Copilot SDK. | - -### Diagrams - -**Deployment overview:** - -![Deployment Diagram](_assets/deployment_diagram.svg) - -**Class relationships:** - -![Class Diagram](_assets/class_diagram.svg) - -### Key Components - -#### `AIChecker` (`src/ai_checker/ai_checker_core.py`) - -Performs the async AI analysis. Responsibilities: - -- Splits artefacts into batches (by count via `--batch-size` and by total - character length via `--max-batch-chars`) -- Processes batches concurrently, rate-limited by an `asyncio.Semaphore` -- Calls `BaseChatModel.with_structured_output(AnalysisResults).ainvoke()` -- Manages the optional result cache (`AnalysisCache`) - -#### `ChatCopilot` (`src/copilot_adapter/copilot_langchain.py`) - -A full `BaseChatModel` implementation backed by the GitHub Copilot SDK CLI -(a Node.js binary). Provides: - -- Standard LangChain message types (system, human, AI, tool) -- Tool calling via `bind_tools()` -- Structured output via `with_structured_output()` -- Native async generation (`_agenerate`) and a sync bridge (`_generate`) -- Pre-flight checks: CLI binary presence, executable bit, `HOME`, proxy vars -- Post-start authentication verification via `get_auth_status()` - -**Why a separate adapter package?** -The `rules_python` wheel packaging strips the executable bit from the -Copilot CLI binary. `ChatCopilot` locates the executable copy created by -the `pip.whl_mods / copy_executables` mechanism and provides clear -diagnostic messages when the environment is misconfigured. The package is -named `copilot_adapter` (not `langchain`) to avoid shadowing the real -`langchain` PyPI package when `imports = ["src"]` is active in Bazel. - -#### `RequirementExtractor` (`src/ai_checker/requirement_extractor.py`) - -Parses TRLC files using the TRLC Python API and returns artefacts as -`dict[str, dict[str, Any]]`. Only objects whose source file resides under -the `--input` directory are analyzed; objects from `--deps` directories are -loaded solely for link resolution. - -#### `AnalysisOrchestrator` (`src/ai_checker/orchestrator.py`) - -Top-level coordinator. Instantiates the extractor, guidelines reader, AI -checker, and result formatter; wires them together; and exposes the CLI -entry point (`main()`). - -#### `GuidelinesReader` (`src/ai_checker/guidelines_reader.py`) - -Reads all `*.md` files from a flat guidelines directory and concatenates -them into the system-message string sent to the AI model. - -#### `ResultFormatter` (`src/ai_checker/result_formatter.py`) - -Formats `AnalysisResults` as JSON or HTML. The HTML report generates -per-guideline markdown subpages linked from the main report. - -### Caching Design - -`AnalysisCache` keys results by `SHA-256(artefacts_json + guidelines + model_name)`. -It is **only** usable via the CLI `--cache` flag. The Bazel rule deliberately -omits `--cache` because Bazel's action cache provides equivalent re-use without -breaking hermeticity. - -### Adding a New Artefact Type - -1. Subclass `ArtefactExtractor` (`src/ai_checker/artefact_extractor.py`) and - implement `extract() -> dict[str, dict[str, Any]]`. -2. Instantiate your extractor in `AnalysisOrchestrator.analyze_directory()` - based on the input file types detected. -3. Add a corresponding Bazel rule in `ai_checker.bzl` following the pattern of - `_trlc_requirements_ai_test_impl`. - -### Updating Python Dependencies - -```bash -# Core + Copilot SDK dependencies -bazel run //validation/ai_checker:requirements.update -``` +Architecture, agent internals, the report pipeline, caching, and extension +points are documented in [DEVELOPMENT.md](DEVELOPMENT.md). diff --git a/validation/ai_checker/_assets/class_diagram.puml b/validation/ai_checker/_assets/class_diagram.puml index 4011e9f5..28a7e1b2 100644 --- a/validation/ai_checker/_assets/class_diagram.puml +++ b/validation/ai_checker/_assets/class_diagram.puml @@ -41,22 +41,30 @@ class AIChecker { -_max_batch_chars: int -_semaphore: asyncio.Semaphore +__init__(model_name, cache_dir, debug_log, batch_size, max_concurrent_requests, max_batch_chars) - +<> analyze(artefacts, guidelines_content, chat_model): AnalysisResults - -_generate_cache_key(artefacts, guidelines_content): str + +<> analyze(artefacts, system_prompt, agent): AnalysisResults + -_generate_cache_key(artefacts, system_prompt): str -_format_artefacts_for_analysis(artefacts): str -_create_batches(artefacts): List[Dict] - -<> _analyze_batch_async(batch_number, artefacts, guidelines_content, chat_model): List[RequirementAnalysis] + -<> _analyze_batch_async(batch_number, artefacts, system_prompt, agent): List[RequirementAnalysis] } -class ChatCopilot <> { +interface AnalysisAgent <> { + +total_costs: float + +total_tokens: int + {abstract} +<> analyze(system_prompt, artefacts_text): AnalysisResults +} + +class CopilotAgent { +model: str +timeout: float - -_bound_tools: list - -_tool_choice: Optional[str] - +_generate(messages, stop, run_manager, **kwargs): ChatResult - +_agenerate(messages, stop, run_manager, **kwargs): ChatResult - +bind_tools(tools, **kwargs): Runnable - +with_structured_output(schema, **kwargs): Runnable + -_manager: CopilotClientManager + +<> analyze(system_prompt, artefacts_text): AnalysisResults +} + +class LangChainAgent { + -_chat_model: BaseChatModel + -_structured: Runnable + +<> analyze(system_prompt, artefacts_text): AnalysisResults } interface ArtefactExtractor <> { @@ -66,88 +74,110 @@ interface ArtefactExtractor <> { class RequirementExtractor { -input_directory: str -dependency_directories: Optional[List[str]] - -symbols: Optional[Symbol_Table] - +__init__(input_directory, dependency_directories) + -req_files: List[str] +extract(): Dict[str, Dict[str, Any]] +parse_trlc_files(): Symbol_Table - +extract_requirements_data(): List[Dict[str, Any]] - +extract_field_value(obj, field_name): Optional[Any] +} + +class ArchitectureExtractor { + -puml_files: List[str] + +extract(): Dict[str, Dict[str, Any]] } class AnalysisOrchestrator { -ai_checker: AIChecker -artefact_extractor: ArtefactExtractor -guidelines_reader: GuidelinesReader - -_chat_model: BaseChatModel + -_agent: AnalysisAgent -model_name: str - -guidelines_path: str -guidelines_content: str + -system_prompt: str -_custom_ai_model: Optional[str] - +__init__(model_name, guidelines_path, cache_dir, debug_log, batch_size, custom_ai_model) - +analyze_directory(input_dir, dependency_dirs): AnalysisResults + +__init__(model_name, guidelines_path, cache_dir, debug_log, batch_size, custom_ai_model, context_files) + +analyze_directory(input_dir, dependency_dirs, req_files, artefact_type, puml_files): AnalysisResults +format_and_output(analysis_results, output_file, html_file, guidelines_output_dir): None } class GuidelinesReader { - -guidelines_dir: str -guidelines: Dict[str, str] - +__init__(guidelines_dir) + +__init__(guidelines_dir, files, extensions) +get_guideline(name): str +get_all_guidelines(): Dict[str, str] - -_load_all_guidelines(): None - -_read_file(file_path): str + +get_combined(): str +} + +class AnalysisReport <> { + +metadata: ReportMetadata + +guidelines: Dict[str, str] + +analyses: List[RequirementAnalysis] } class ResultFormatter { - -results: AnalysisResults - -model_name: str - -guidelines_reader: Optional[GuidelinesReader] + -report: AnalysisReport -guidelines_output_dir: Optional[str] - -original_requirements: Optional[Dict] - -git_hash: str - -timestamp: str - +__init__(analysis_results, model_name, guidelines_reader, guidelines_output_dir, original_requirements) + +__init__(analysis_results, model_name, guidelines_reader, guidelines_output_dir, original_requirements, artefact_type) +output(file_path): None - -_print_to_stdout(): None - -_write_json(path): None - -_write_html(path): None - -_generate_html_report(): str - -_generate_guideline_pages(main_report_path): None - {static} -_get_git_hash(): str - {static} -_get_timestamp(): str } +interface ReportRenderer <> { + +extension: str + {abstract} +render(report): str + +write_extras(report, out_path): None +} + +class JsonRenderer +class HtmlRenderer +class RstRenderer + AnalysisResults o-- RequirementAnalysis AnalysisCache ..> AnalysisResults AIChecker o-- AnalysisCache AIChecker ..> AnalysisResults +AIChecker --> AnalysisAgent: analyze(system_prompt, artefacts) +AnalysisAgent <|.. CopilotAgent +AnalysisAgent <|.. LangChainAgent +LangChainAgent o-- "any" BaseChatModel ArtefactExtractor <|.. RequirementExtractor +ArtefactExtractor <|.. ArchitectureExtractor AnalysisOrchestrator o-- AIChecker AnalysisOrchestrator o-- GuidelinesReader AnalysisOrchestrator o-- ArtefactExtractor -AnalysisOrchestrator --> ChatCopilot: creates (default) +AnalysisOrchestrator --> CopilotAgent: creates (default) AnalysisOrchestrator ..> AnalysisResults ResultFormatter ..> AnalysisResults -ResultFormatter o-- "optional" GuidelinesReader +ResultFormatter o-- AnalysisReport +ResultFormatter ..> ReportRenderer: dispatch by extension +AnalysisReport o-- RequirementAnalysis +ReportRenderer <|.. JsonRenderer +ReportRenderer <|.. HtmlRenderer +ReportRenderer <|.. RstRenderer +ReportRenderer ..> AnalysisReport note right of AnalysisOrchestrator - Default: creates ChatCopilot. - Custom: loads ai_model.py - at runtime for alternative - AI model implementations. + Default: creates CopilotAgent. + Custom: loads ai_model.py at + runtime; its create_agent() + returns any AnalysisAgent. +end note + +note right of AnalysisAgent + Core depends only on this + interface (no LangChain). + Default impl talks to the + Copilot SDK directly. end note -note right of ChatCopilot - LangChain BaseChatModel wrapper - for the GitHub Copilot SDK. - Uses Copilot CLI (Node.js binary) - for API communication. +note right of CopilotAgent + Direct GitHub Copilot SDK + (Node.js CLI), no LangChain. + Reports tokens / credits. end note note right of ArtefactExtractor Interface allows plugging different extractors - (TRLC, code, etc.) + (TRLC requirements, raw + PlantUML architecture, ...) end note @enduml diff --git a/validation/ai_checker/_assets/class_diagram.svg b/validation/ai_checker/_assets/class_diagram.svg index e773e88c..555b454e 100644 --- a/validation/ai_checker/_assets/class_diagram.svg +++ b/validation/ai_checker/_assets/class_diagram.svg @@ -1 +1,224 @@ -«BaseModel»RequirementAnalysisrequirement_id: strdescription: strfindings: List[str]suggestions: List[str]score: float«BaseModel»AnalysisResultsanalyses: List[RequirementAnalysis]AnalysisCache_cache_dir: Path__init__(cache_dir)get(cache_hash): Optional[AnalysisResults]set(cache_hash, results): Noneis_enabled(): boolAIChecker_model_name: str_cache: AnalysisCache_batch_size: int_max_concurrent_requests: int_max_batch_chars: int_semaphore: asyncio.Semaphore__init__(model_name, cache_dir, debug_log, batch_size, max_concurrent_requests, max_batch_chars)«async» analyze(artefacts, guidelines_content, chat_model): AnalysisResults_generate_cache_key(artefacts, guidelines_content): str_format_artefacts_for_analysis(artefacts): str_create_batches(artefacts): List[Dict]«async» _analyze_batch_async(batch_number, artefacts, guidelines_content, chat_model): List[RequirementAnalysis]«BaseChatModel»ChatCopilotmodel: strtimeout: float_bound_tools: list_tool_choice: Optional[str]_generate(messages, stop, run_manager, **kwargs): ChatResult_agenerate(messages, stop, run_manager, **kwargs): ChatResultbind_tools(tools, **kwargs): Runnablewith_structured_output(schema, **kwargs): Runnable«interface»ArtefactExtractorextract(): Dict[str, Dict[str, Any]]RequirementExtractorinput_directory: strdependency_directories: Optional[List[str]]symbols: Optional[Symbol_Table]__init__(input_directory, dependency_directories)extract(): Dict[str, Dict[str, Any]]parse_trlc_files(): Symbol_Tableextract_requirements_data(): List[Dict[str, Any]]extract_field_value(obj, field_name): Optional[Any]AnalysisOrchestratorai_checker: AICheckerartefact_extractor: ArtefactExtractorguidelines_reader: GuidelinesReader_chat_model: BaseChatModelmodel_name: strguidelines_path: strguidelines_content: str_custom_ai_model: Optional[str]__init__(model_name, guidelines_path, cache_dir, debug_log, batch_size, custom_ai_model)analyze_directory(input_dir, dependency_dirs): AnalysisResultsformat_and_output(analysis_results, output_file, html_file, guidelines_output_dir): NoneGuidelinesReaderguidelines_dir: strguidelines: Dict[str, str]__init__(guidelines_dir)get_guideline(name): strget_all_guidelines(): Dict[str, str]_load_all_guidelines(): None_read_file(file_path): strResultFormatterresults: AnalysisResultsmodel_name: strguidelines_reader: Optional[GuidelinesReader]guidelines_output_dir: Optional[str]original_requirements: Optional[Dict]git_hash: strtimestamp: str__init__(analysis_results, model_name, guidelines_reader, guidelines_output_dir, original_requirements)output(file_path): None_print_to_stdout(): None_write_json(path): None_write_html(path): None_generate_html_report(): str_generate_guideline_pages(main_report_path): None_get_git_hash(): str_get_timestamp(): strDefault: creates ChatCopilot.Custom: loads ai_model.pyat runtime for alternativeAI model implementations.LangChain BaseChatModel wrapperfor the GitHub Copilot SDK.Uses Copilot CLI (Node.js binary)for API communication.Interface allows pluggingdifferent extractors(TRLC, code, etc.)creates (default)optional \ No newline at end of file +«BaseModel»RequirementAnalysisrequirement_id: strdescription: strfindings: List[str]suggestions: List[str]score: float«BaseModel»AnalysisResultsanalyses: List[RequirementAnalysis]AnalysisCache_cache_dir: Path__init__(cache_dir)get(cache_hash): Optional[AnalysisResults]set(cache_hash, results): Noneis_enabled(): boolAIChecker_model_name: str_cache: AnalysisCache_batch_size: int_max_concurrent_requests: int_max_batch_chars: int_semaphore: asyncio.Semaphore__init__(model_name, cache_dir, debug_log, batch_size, max_concurrent_requests, max_batch_chars)«async» analyze(artefacts, system_prompt, agent): AnalysisResults_generate_cache_key(artefacts, system_prompt): str_format_artefacts_for_analysis(artefacts): str_create_batches(artefacts): List[Dict]«async» _analyze_batch_async(batch_number, artefacts, system_prompt, agent): List[RequirementAnalysis]«interface»AnalysisAgenttotal_costs: floattotal_tokens: int«async» analyze(system_prompt, artefacts_text): AnalysisResultsCopilotAgentmodel: strtimeout: float_manager: CopilotClientManager«async» analyze(system_prompt, artefacts_text): AnalysisResultsLangChainAgent_chat_model: BaseChatModel_structured: Runnable«async» analyze(system_prompt, artefacts_text): AnalysisResults«interface»ArtefactExtractorextract(): Dict[str, Dict[str, Any]]RequirementExtractorinput_directory: strdependency_directories: Optional[List[str]]req_files: List[str]extract(): Dict[str, Dict[str, Any]]parse_trlc_files(): Symbol_TableArchitectureExtractorpuml_files: List[str]extract(): Dict[str, Dict[str, Any]]AnalysisOrchestratorai_checker: AICheckerartefact_extractor: ArtefactExtractorguidelines_reader: GuidelinesReader_agent: AnalysisAgentmodel_name: strguidelines_content: strsystem_prompt: str_custom_ai_model: Optional[str]__init__(model_name, guidelines_path, cache_dir, debug_log, batch_size, custom_ai_model, context_files)analyze_directory(input_dir, dependency_dirs, req_files, artefact_type, puml_files): AnalysisResultsformat_and_output(analysis_results, output_file, html_file, guidelines_output_dir): NoneGuidelinesReaderguidelines: Dict[str, str]__init__(guidelines_dir, files, extensions)get_guideline(name): strget_all_guidelines(): Dict[str, str]get_combined(): str«BaseModel»AnalysisReportmetadata: ReportMetadataguidelines: Dict[str, str]analyses: List[RequirementAnalysis]ResultFormatterreport: AnalysisReportguidelines_output_dir: Optional[str]__init__(analysis_results, model_name, guidelines_reader, guidelines_output_dir, original_requirements, artefact_type)output(file_path): None«interface»ReportRendererextension: strrender(report): strwrite_extras(report, out_path): NoneJsonRendererHtmlRendererRstRendererBaseChatModelDefault: creates CopilotAgent.Custom: loads ai_model.py atruntime; its create_agent()returns any AnalysisAgent.Core depends only on thisinterface (no LangChain).Default impl talks to theCopilot SDK directly.Direct GitHub Copilot SDK(Node.js CLI), no LangChain.Reports tokens / credits.Interface allows pluggingdifferent extractors(TRLC requirements, rawPlantUML architecture, ...)analyze(system_prompt, artefacts)anycreates (default)dispatch by extension \ No newline at end of file diff --git a/validation/ai_checker/_assets/deployment_diagram.puml b/validation/ai_checker/_assets/deployment_diagram.puml index c6f25c3c..eb333940 100644 --- a/validation/ai_checker/_assets/deployment_diagram.puml +++ b/validation/ai_checker/_assets/deployment_diagram.puml @@ -16,28 +16,30 @@ skinparam packageStyle rectangle interface "TRLC Python API" as TRLCAPI -interface "LangChain\nBaseChatModel" as LangChain +interface "AnalysisAgent\n(interface)" as Agent package "Input Artifacts" { artifact "TRLC\nRequirements" as ReqFile + artifact "Architecture\n(PlantUML)" as PumlFile artifact "Guidelines\n(Markdown)" as GuideFile + artifact "Context\n(md / puml)" as ContextFile } -package "src/ai_checker/ (Core Framework)" as CorePkg { +package "src/ai_checker/ (Core Framework — no LangChain)" as CorePkg { component "Orchestrator\n(py_binary)" as Orchestrator component "AIChecker\n(Async)" as AIChecker component "Requirement\nExtractor" as Extractor + component "Architecture\nExtractor" as ArchExtractor component "Result\nFormatter" as Formatter component "Guidelines\nReader" as GReader component "Analysis\nCache" as Cache } -package "src/copilot_adapter/ (LangChain Integration)" as LangChainPkg { - component "ChatCopilot\n(BaseChatModel)" as ChatCopilot +package "src/ai_checker/agents/ (AI backends)" as AdapterPkg { + component "CopilotAgent\n(default, direct SDK)" as CopilotAgent + component "LangChainAgent\n(optional)" as LCAgent } - - package "Output Artifacts" { artifact "HTML\nReport" as HtmlReport artifact "JSON\nResults" as JsonResults @@ -48,39 +50,42 @@ package "AI Services" { cloud "Custom AI Service\n(via ai_model.py)" as CustomCloud } -package "Bazel Configuration" { - component "--config=copilot\n(.bazelrc.ai_checker)" as CopilotConfig +package "Test Environment" { + component "RunEnvironmentInfo\n(inherited_environment)" as CopilotConfig } ReqFile --> TRLCAPI: parsed via TRLCAPI --> Extractor: symbol table +PumlFile --> ArchExtractor: raw .puml text Extractor --> Orchestrator: artefacts\n(Dict format) +ArchExtractor --> Orchestrator: artefacts\n(Dict format) GuideFile --> GReader: loads -GReader --> Orchestrator: guidelines content +ContextFile --> GReader: loads +GReader --> Orchestrator: guidelines + context\n(system prompt) Orchestrator --> AIChecker: async analyze\nwith batches AIChecker --> Cache: optional\n(CLI only) -AIChecker --> LangChain: async ainvoke\n(concurrent) -ChatCopilot ..|> LangChain: implements -ChatCopilot --> CopilotAPI: Copilot SDK\n(Node.js CLI) - -LangChain --> CustomCloud: custom provider +AIChecker --> Agent: analyze(system_prompt,\nartefacts)\n(concurrent) +CopilotAgent ..|> Agent: implements +LCAgent ..|> Agent: implements +CopilotAgent --> CopilotAPI: Copilot SDK\n(Node.js CLI) +LCAgent --> CustomCloud: any LangChain\nBaseChatModel -LangChain ..> AIChecker: structured output\n(AnalysisResults) +Agent ..> AIChecker: AnalysisResults AIChecker ..> Orchestrator: AnalysisResults Orchestrator --> Formatter: pass results Formatter --> HtmlReport: generates Formatter --> JsonResults: generates -CopilotConfig ..> ChatCopilot: env vars\n(auth + proxy) +CopilotConfig ..> CopilotAgent: inherited env\n(auth + proxy) -note right of ChatCopilot - LangChain BaseChatModel wrapper for - the GitHub Copilot SDK. - Supports: messages, tool calling, - structured output, async/sync. +note right of CopilotAgent + Default backend. Talks to the + GitHub Copilot SDK directly, + no LangChain. Reports usage + (tokens / credits). end note note as N1 diff --git a/validation/ai_checker/_assets/deployment_diagram.svg b/validation/ai_checker/_assets/deployment_diagram.svg index 5cf6d8e9..d244fb47 100644 --- a/validation/ai_checker/_assets/deployment_diagram.svg +++ b/validation/ai_checker/_assets/deployment_diagram.svg @@ -1 +1 @@ -Input Artifactssrc/ai_checker/ (Core Framework)src/copilot_adapter/ (LangChain Integration)Output ArtifactsAI ServicesBazel ConfigurationTRLCRequirementsGuidelines(Markdown)Orchestrator(py_binary)AIChecker(Async)RequirementExtractorResultFormatterGuidelinesReaderAnalysisCacheChatCopilot(BaseChatModel)HTMLReportJSONResultsGitHub Copilot API(default)Custom AI Service(via ai_model.py)--config=copilot(.bazelrc.ai_checker)TRLC Python APILangChainBaseChatModelLangChain BaseChatModel wrapper forthe GitHub Copilot SDK.Supports: messages, tool calling,structured output, async/sync.AIChecker - Async Features:- Concurrent batch processing- Rate limiting (Semaphore)- Smart batching (count + size)Cache:Intentionally disabled for Bazel builds(Bazel has its own action cache).Only available via CLI: --cache <dir>parsed viasymbol tableartefacts(Dict format)loadsguidelines contentasync analyzewith batchesAnalysisResultsoptional(CLI only)async ainvoke(concurrent)structured output(AnalysisResults)implementsCopilot SDK(Node.js CLI)custom providerpass resultsgeneratesgeneratesenv vars(auth + proxy) \ No newline at end of file +Input Artifactssrc/ai_checker/ (Core Framework — no LangChain)src/ai_checker/agents/ (AI backends)Output ArtifactsAI ServicesTest EnvironmentTRLCRequirementsArchitecture(PlantUML)Guidelines(Markdown)Context(md / puml)Orchestrator(py_binary)AIChecker(Async)RequirementExtractorArchitectureExtractorResultFormatterGuidelinesReaderAnalysisCacheCopilotAgent(default, direct SDK)LangChainAgent(optional)HTMLReportJSONResultsGitHub Copilot API(default)Custom AI Service(via ai_model.py)RunEnvironmentInfo(inherited_environment)TRLC Python APIAnalysisAgent(interface)Default backend. Talks to theGitHub Copilot SDK directly,no LangChain. Reports usage(tokens / credits).AIChecker - Async Features:- Concurrent batch processing- Rate limiting (Semaphore)- Smart batching (count + size)Cache:Intentionally disabled for Bazel builds(Bazel has its own action cache).Only available via CLI: --cache <dir>parsed viasymbol tableraw .puml textartefacts(Dict format)artefacts(Dict format)loadsloadsguidelines + context(system prompt)async analyzewith batchesAnalysisResultsoptional(CLI only)analyze(system_prompt,artefacts)(concurrent)AnalysisResultsimplementsimplementsCopilot SDK(Node.js CLI)any LangChainBaseChatModelpass resultsgeneratesgeneratesinherited env(auth + proxy) \ No newline at end of file diff --git a/validation/ai_checker/ai_checker.bzl b/validation/ai_checker/ai_checker.bzl index d0816b75..939f28d2 100644 --- a/validation/ai_checker/ai_checker.bzl +++ b/validation/ai_checker/ai_checker.bzl @@ -23,19 +23,63 @@ load("//bazel/rules/rules_score:providers.bzl", "ArchitecturalDesignInfo") # Shared implementation # ============================================================================ -def _run_ai_analysis(ctx, analysis_files, all_input_files, input_dirs, dep_dirs, req_files = None): +# Default tags applied to every AI test. The AI analysis runs at test time and +# performs a non-hermetic network call to GitHub Copilot using credentials from +# the user's environment, so the test must escape Bazel's sandbox. "external" +# prevents result caching (the AI response is non-deterministic). +_AI_TEST_DEFAULT_TAGS = ["no-sandbox", "requires-network", "external"] + +# Environment variables the test inherits from the invoking client environment. +# These are baked into the target via RunEnvironmentInfo so consumers do not +# need a --config=copilot / --test_env flag. HOME is essential: the test runner +# otherwise resets HOME to $TEST_TMPDIR (per the Bazel Test Encyclopedia), +# which hides the Copilot CLI's ~/.copilot/config.json credentials. The proxy +# variables are inherited so the call works behind a corporate proxy. +_AI_TEST_INHERITED_ENV = [ + "HOME", + "COPILOT_GITHUB_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] + +def _shell_quote(value): + if value == "": + return "''" + return "'" + value.replace("'", "'\"'\"'") + "'" + +def _short_dir(f): + """Return the runfiles-relative directory of a File (short_path dirname).""" + sp = f.short_path + if "/" in sp: + return sp.rsplit("/", 1)[0] + return "." + +def _run_ai_analysis(ctx, analysis_files, all_input_files, dep_dirs, req_files = None, artefact_type = "requirements"): """Common implementation for all AI artefact analysis test rules. + The AI analysis runs **at test time** (not as a build action): the test + executable launches the orchestrator from the runfiles tree, so the + non-hermetic network call and its credentials live in the test phase. The + required environment is inherited automatically via RunEnvironmentInfo, and + reports are written to the test's undeclared-outputs directory. + Args: ctx: Rule context. analysis_files: Files to analyze (direct inputs). - all_input_files: All files needed as action inputs (incl. deps for resolution). - input_dirs: Dict of directories containing analysis files. - dep_dirs: Dict of dependency directories (for link resolution). + all_input_files: All files needed at runtime (incl. deps for resolution). + dep_dirs: Dict of runfiles-relative dependency directories (link resolution). req_files: Optional list of individual files to register with TRLC instead of scanning the entire input directory. When set, only these files are - parsed; other files present in the same directory are ignored. This - avoids picking up unreferenced files that may fail TRLC validation. + parsed and they also define the grading scope, so requirements spread + across several directories are all graded. Passing them explicitly + also avoids picking up unreferenced files that may fail TRLC validation. + artefact_type: "requirements" (TRLC) or "architecture" (raw PlantUML). Returns: List of providers (DefaultInfo). @@ -43,133 +87,115 @@ def _run_ai_analysis(ctx, analysis_files, all_input_files, input_dirs, dep_dirs, if not analysis_files: fail("No artefact files found for analysis") - # Declare outputs - html_report = ctx.actions.declare_file("{}_analysis.html".format(ctx.attr.name)) - json_report = ctx.actions.declare_file("{}_analysis.json".format(ctx.attr.name)) - guidelines_output_dir = ctx.actions.declare_directory("guidelines") - debug_log = ctx.actions.declare_file("{}_debug.log".format(ctx.attr.name)) - - # Collect guideline files from the filegroup + # Collect guideline / context / project-guideline files. guideline_files = ctx.files.guidelines + context_files = ctx.files.context + project_guideline_files = ctx.files._project_guidelines - # Determine input and guidelines directories - input_dir = analysis_files[0].dirname - guidelines_dir = guideline_files[0].dirname if guideline_files else None - - # Build arguments for the orchestrator - args = ctx.actions.args() - args.add("--input", input_dir) + # Build the orchestrator argument list. The orchestrator runs from the + # runfiles root at test time, so every file is referenced by its + # workspace-relative short_path. + args = ["--artefact-type", artefact_type] for dep_dir in dep_dirs.keys(): - args.add("--deps", dep_dir) + args += ["--deps", dep_dir] - for extra_dir in input_dirs.keys(): - if extra_dir != input_dir: - args.add("--deps", extra_dir) - - # When individual req files are provided, pass them explicitly so the - # extractor registers only those files and ignores other files present in - # the same directory (e.g. files not declared in Bazel srcs that may fail - # TRLC validation). + # When individual req files are provided, pass them explicitly. The + # extractor then registers only those files (ignoring other files in the + # same directory that may fail TRLC validation) and grades exactly that + # set, so requirements located in different directories are all covered. if req_files: for f in req_files: - args.add("--req-file", f) - - args.add("--output", json_report.path) - args.add("--html", html_report.path) - args.add("--guidelines-output", guidelines_output_dir.path) - - if guidelines_dir: - args.add("--guidelines", guidelines_dir) + args += ["--req-file", f.short_path] + + # For architecture, pass the raw PlantUML source files explicitly (the + # orchestrator reads them as text). + if artefact_type == "architecture": + for f in analysis_files: + args += ["--puml-file", f.short_path] + + # Background context (markdown / plantuml) injected as read-only material. + for f in context_files: + args += ["--context-file", f.short_path] + + # Project-specific guidelines (graded) layered on top of the general and + # type guidelines. Sourced from a label_flag so a consumer repo can set + # them once in .bazelrc instead of on every target. + for f in project_guideline_files: + args += ["--project-guidelines", f.short_path] + + # Pass each guideline file explicitly rather than a directory: the default + # guideline sets span multiple directories (e.g. general.md plus a + # requirements/ or architecture/ subdirectory) and the orchestrator scans a + # guidelines directory non-recursively, so a single derived directory would + # silently drop guidelines from the other directories. + for f in guideline_files: + args += ["--guidelines-file", f.short_path] if ctx.attr.model: - args.add("--model", ctx.attr.model) + args += ["--model", ctx.attr.model] if ctx.attr.batch_size > 0: - args.add("--batch-size", str(ctx.attr.batch_size)) + args += ["--batch-size", str(ctx.attr.batch_size)] - # NOTE: --cache is intentionally NOT passed. Bazel actions are - # already cached by Bazel's action cache; an additional Python-level - # cache would break hermeticity. The --cache flag is only available - # for direct CLI invocations (python orchestrator.py --cache ). + # NOTE: --cache is intentionally NOT passed. The --cache flag is only + # available for direct CLI invocations (python orchestrator.py --cache). - # Prepare action inputs (include custom ai_model if provided) - action_inputs = all_input_files + guideline_files - if ctx.attr._custom_ai_model: - custom_ai_model_file = ctx.attr._custom_ai_model[DefaultInfo].files.to_list() - if custom_ai_model_file: - action_inputs.extend(custom_ai_model_file) - args.add("--custom-ai-model", custom_ai_model_file[0].path) - - # Add debug log output for Bazel output_groups - args.add("--debug-log", debug_log.path) - args.add("--verbose") - - ctx.actions.run( - executable = ctx.executable._orchestrator, - inputs = depset(direct = action_inputs), - outputs = [json_report, html_report, guidelines_output_dir, debug_log], - arguments = [args], - progress_message = "Analyzing artefacts with AI for {}".format(ctx.attr.name), - # NOTE: no-sandbox is required because the GitHub Copilot CLI needs - # outbound network access (api.github.com) and the user's $HOME - # directory (for stored OAuth credentials), both of which Bazel's - # sandbox blocks. This is an inherent trade-off of using an external - # AI service from a Bazel action. Hermeticity is partially preserved - # by Bazel's own action-cache keying on declared inputs. - execution_requirements = {"no-sandbox": "1"}, - use_default_shell_env = True, + # Files that must be present in the test runfiles. + runtime_files = ( + all_input_files + guideline_files + context_files + project_guideline_files ) - # Test executable — validates the JSON report score against the threshold - test_executable = ctx.actions.declare_file("{}_test_executable".format(ctx.attr.name)) - - command = """#!/bin/bash -set -e -set -o pipefail - -json_path="{json}" - -if [ ! -f "$json_path" ] || [ ! -s "$json_path" ]; then - echo "ERROR: JSON report was not generated or is empty" - exit 1 -fi - -average=$(python3 -c " -import json, pathlib, sys -data = json.loads(pathlib.Path(sys.argv[1]).read_text()) -scores = [a['score'] for a in data.get('analyses', [])] -print(f'{{sum(scores)/len(scores):.2f}}' if scores else '0') -" "$json_path") - -threshold="{threshold}" - -if (( $(echo "$average >= $threshold" | bc -l) )); then - echo "AI analysis complete. Average score: $average (threshold: $threshold)" - exit 0 -else - echo "ERROR: Average score $average is below threshold $threshold" - exit 1 -fi -""".format(json = json_report.short_path, threshold = ctx.attr.score_threshold) + # Optional custom AI backend supplied by the consumer repo. + if ctx.attr._custom_ai_model: + custom_ai_model_files = ctx.attr._custom_ai_model[DefaultInfo].files.to_list() + if custom_ai_model_files: + runtime_files = runtime_files + custom_ai_model_files + args += ["--custom-ai-model", custom_ai_model_files[0].short_path] + + # Generate the test launcher. Per the Bazel Test Encyclopedia, a test runs + # with its working directory set to $TEST_SRCDIR/$TEST_WORKSPACE (the + # runfiles root), so the workspace-relative artefact paths baked above + # resolve directly and no runfiles probing is required. The orchestrator + # writes its reports into $TEST_UNDECLARED_OUTPUTS_DIR itself, so the + # launcher only has to exec it with the computed arguments. + name = ctx.attr.name + launcher_content = ( + "#!/usr/bin/env bash\n" + + "set -euo pipefail\n\n" + + "# A test starts in $TEST_SRCDIR/$TEST_WORKSPACE (the runfiles root), so\n" + + "# the workspace-relative paths baked below resolve as-is. The\n" + + "# orchestrator writes its reports into the test's undeclared-outputs\n" + + "# directory automatically (Bazel zips it into\n" + + "# bazel-testlogs/.../test.outputs/outputs.zip).\n" + + "exec \"./" + ctx.executable._orchestrator.short_path + "\" \\\n" + + " " + " ".join([_shell_quote(a) for a in args]) + " \\\n" + + " --verbose \\\n" + + " --score-threshold " + _shell_quote(ctx.attr.score_threshold) + "\n" + ) + launcher = ctx.actions.declare_file("{}_ai_test.sh".format(name)) ctx.actions.write( - output = test_executable, - content = command, + output = launcher, + content = launcher_content, is_executable = True, ) + # Everything the orchestrator needs at test time travels in the runfiles: + # the orchestrator binary (+ its own runfiles) and all artefact / guideline + # inputs. + runfiles = ctx.runfiles( + files = [ctx.executable._orchestrator] + runtime_files, + ).merge(ctx.attr._orchestrator[DefaultInfo].default_runfiles) + return [ DefaultInfo( - runfiles = ctx.runfiles( - files = [json_report, html_report, guidelines_output_dir], - ), - files = depset([json_report, html_report, guidelines_output_dir]), - executable = test_executable, - ), - OutputGroupInfo( - debug = depset([debug_log]), + executable = launcher, + runfiles = runfiles, ), + # Bake the required environment-variable inheritance into the target so + # `bazel test //...` works without a --config=copilot / --test_env flag. + RunEnvironmentInfo(inherited_environment = _AI_TEST_INHERITED_ENV), ] # Attributes shared by all AI test rules @@ -186,16 +212,22 @@ _COMMON_AI_TEST_ATTRS = { doc = "Number of artefacts to process per batch (0 = all at once).", default = 0, ), + "context": attr.label( + doc = "Optional filegroup of background-context files (.md / .puml) " + + "passed to the AI as read-only reference material.", + allow_files = [".md", ".puml"], + default = None, + ), "_custom_ai_model": attr.label( doc = "Custom ai_model.py file (optional, provided by consumer repo).", default = None, allow_single_file = [".py"], ), "_orchestrator": attr.label( - doc = "Orchestrator binary.", + doc = "Orchestrator binary (runs at test time, hence target config).", default = "//validation/ai_checker:orchestrator", executable = True, - cfg = "exec", + cfg = "target", ), } @@ -207,7 +239,6 @@ def _trlc_requirements_ai_test_impl(ctx): """Extract TRLC artefacts from providers and delegate to shared analysis.""" analysis_files = [] all_files = [] - input_dirs = {} dep_dirs = {} for req in ctx.attr.reqs: @@ -215,18 +246,16 @@ def _trlc_requirements_ai_test_impl(ctx): direct_reqs = trlc_provider.reqs.to_list() analysis_files.extend(direct_reqs) - for f in direct_reqs: - input_dirs[f.dirname] = True dep_reqs = trlc_provider.deps.to_list() spec_files = trlc_provider.spec.to_list() all_files.extend(direct_reqs + dep_reqs + spec_files) for f in dep_reqs + spec_files: - dep_dirs[f.dirname] = True + dep_dirs[_short_dir(f)] = True - return _run_ai_analysis(ctx, analysis_files, all_files, input_dirs, dep_dirs, req_files = analysis_files) + return _run_ai_analysis(ctx, analysis_files, all_files, dep_dirs, req_files = analysis_files) -trlc_requirements_ai_test = rule( +_trlc_requirements_ai_test = rule( implementation = _trlc_requirements_ai_test_impl, attrs = dict(_COMMON_AI_TEST_ATTRS, **{ "reqs": attr.label_list( @@ -239,30 +268,61 @@ trlc_requirements_ai_test = rule( default = "//validation/ai_checker:default_guidelines", allow_files = True, ), + "_project_guidelines": attr.label( + doc = "Project-specific guideline files (graded), settable once via " + + "the //validation/ai_checker:project_guidelines flag.", + default = "//validation/ai_checker:project_guidelines", + allow_files = True, + ), }), test = True, toolchains = [], fragments = ["platform"], ) +def trlc_requirements_ai_test(name, **kwargs): + """AI review of TRLC requirements (runs the analysis at test time). + + The AI call is non-hermetic (network + credentials), so default tags mark + the test as un-sandboxed and network-dependent. Any caller-supplied tags + are merged on top. + """ + tags = kwargs.pop("tags", []) + _trlc_requirements_ai_test( + name = name, + tags = _AI_TEST_DEFAULT_TAGS + [t for t in tags if t not in _AI_TEST_DEFAULT_TAGS], + **kwargs + ) + # ============================================================================ # Architecture AI Test # ============================================================================ def _architecture_ai_test_impl(ctx): - """Extract architecture artefacts from providers and delegate to shared analysis.""" + """Extract architecture artefacts from providers and delegate to shared analysis. + + Architecture review reads the raw PlantUML *source* (not the parsed + FlatBuffers binaries in ArchitecturalDesignInfo.static/dynamic). The design + target's DefaultInfo carries the raw .puml source files. + """ analysis_files = [] - input_dirs = {} + # The "designs" attr requires ArchitecturalDesignInfo, so only architectural + # designs are accepted; the AI reads the raw .puml sources from DefaultInfo. for design in ctx.attr.designs: - design_info = design[ArchitecturalDesignInfo] - for f in design_info.static.to_list() + design_info.dynamic.to_list(): - analysis_files.append(f) - input_dirs[f.dirname] = True - - return _run_ai_analysis(ctx, analysis_files, analysis_files, input_dirs, {}) + for f in design[DefaultInfo].files.to_list(): + if f.extension == "puml": + analysis_files.append(f) + + return _run_ai_analysis( + ctx, + analysis_files, + analysis_files, + {}, + artefact_type = "architecture", + ) -architecture_ai_test = rule( +_architecture_ai_test = rule( implementation = _architecture_ai_test_impl, attrs = dict(_COMMON_AI_TEST_ATTRS, **{ "designs": attr.label_list( @@ -275,8 +335,28 @@ architecture_ai_test = rule( default = "//validation/ai_checker:default_architecture_guidelines", allow_files = True, ), + "_project_guidelines": attr.label( + doc = "Project-specific guideline files (graded), settable once via " + + "the //validation/ai_checker:project_architecture_guidelines flag.", + default = "//validation/ai_checker:project_architecture_guidelines", + allow_files = True, + ), }), test = True, toolchains = [], fragments = ["platform"], ) + +def architecture_ai_test(name, **kwargs): + """AI review of PlantUML architecture (runs the analysis at test time). + + The AI call is non-hermetic (network + credentials), so default tags mark + the test as un-sandboxed and network-dependent. Any caller-supplied tags + are merged on top. + """ + tags = kwargs.pop("tags", []) + _architecture_ai_test( + name = name, + tags = _AI_TEST_DEFAULT_TAGS + [t for t in tags if t not in _AI_TEST_DEFAULT_TAGS], + **kwargs + ) diff --git a/validation/ai_checker/guidelines/architecture/architecture_guidelines.md b/validation/ai_checker/guidelines/architecture/architecture_guidelines.md new file mode 100644 index 00000000..0a76de39 --- /dev/null +++ b/validation/ai_checker/guidelines/architecture/architecture_guidelines.md @@ -0,0 +1,42 @@ + + +# Architecture Design Guidelines + +General guidelines for reviewing architectural design diagrams (PlantUML). Each +artefact is the raw PlantUML source of a static (component / class) or dynamic +(sequence) diagram. Project-specific details (e.g. the available architecture +levels) are supplied separately by the project guidelines. + +## Quality Criteria + +A well-formed architectural diagram is: + +- **Named** — every component, interface, participant, and relation has an explicit, meaningful name (no anonymous or placeholder names). +- **Interface-explicit** — components communicate through declared interfaces, not direct undocumented coupling. +- **Consistent** — element names match the corresponding requirements and other diagrams; no contradictions across static and dynamic views. +- **Traceable** — components and interfaces map to feature/component requirements they realize. +- **Layered** — dependencies flow in one direction; no cyclic dependencies between components. +- **Cohesive** — each component has a single, clear responsibility. +- **Complete** — every relation referenced in a sequence diagram exists in the corresponding component/class diagram. + +### Avoid + +- Unconnected or orphan elements (declared but never related). +- Direct access bypassing declared interfaces. +- Cyclic dependencies between components. +- Overloaded components with unrelated responsibilities. +- Relations without stereotypes/labels where the semantics are not obvious. +- Mismatched names between the architecture and the requirements it realizes. + +## Static vs Dynamic Consistency diff --git a/validation/ai_checker/guidelines/general.md b/validation/ai_checker/guidelines/general.md index 2918e7c6..f1a1204c 100644 --- a/validation/ai_checker/guidelines/general.md +++ b/validation/ai_checker/guidelines/general.md @@ -11,29 +11,71 @@ SPDX-License-Identifier: Apache-2.0 ----------------------------------------------------------------------------- --> -- Markdown Expressions (Link: "[label](http://example.com)") -- RST Expressions (Link: ".. _a link: https://domain.invalid/" or `a link`_ ) -- Expression which are provided as comment or via `` as fully defined. +# General Review Guidelines -Do a requirements review of each single requirement: -- Review the requirements according to the guidelines, for the sentence template also take into consideration optional and mandatory parts of a sentence. -- Include the requirement type in the analysis. -- Accept expressions as defined if in doubt +Common review methodology that applies to every artefact type (requirements, +architecture, …). Element-type-specific criteria are supplied by the matching +type guideline, and project-specific rules are supplied by the project +guidelines; this document only defines *how* to review and *how* to report. + +## Expression Syntax + +Treat the following as already defined; do not flag them as undefined: + +- Markdown expressions (link: `[label](http://example.com)`) +- RST expressions (link: `.. _a link: https://domain.invalid/` or `` `a link`_ ``) +- Expressions provided as a comment or inside `` `` `` as fully defined. + +## Applicable Standards + +The project develops safety-related software under **ISO 26262** (road-vehicle +functional safety). Treat ISO 26262 as the governing functional-safety standard: + +- Terms such as *safety-certified*, *safety-related*, *freedom from + interference*, and *ASIL* (incl. ASIL A–D / QM) are defined by ISO 26262 and + shall **not** be flagged as vague or undefined. +- Do not flag a safety requirement merely because it does not restate the + standard or an ASIL level; the applicable standard and integrity level are + established here at project level. + +## Review Methodology + +Review each element on its own: + +- Evaluate the element against the applicable type and project guidelines. +- Accept expressions as defined if in doubt. Do not: -- analyze the hierarchy of the requirements or relations between them (e.g. stated via attribute parent) -- mention any findings of fully defined items -- take the sentence template to strict -Score the requirement from 0-10 where: -- 0-3: Critical issues, requirement needs major rework +- analyze the hierarchy of elements or relations between them (e.g. a parent + attribute); +- mention any findings for fully defined items; +- apply any template too strictly. + +## Scoring + +Score each element from 0-10 where: + +- 0-3: Critical issues, the element needs major rework - 4-6: Moderate issues, improvement needed - 7-8: Good quality with minor improvements possible - 9-10: Excellent quality, meets professional standards -Analysis Result: -- Provide specific, actionable findings and refer it to a specific keyword from the document. (e.g. *Vague* Requirement contains ....) -- Only point out findings, don´t mention if something is particularly well defined -- Categorize findings in major and minor -- Provide an overall scoring for the requirement -- If required use HTML formatting +The score **must be consistent with the findings**: + +- If an element has **no findings and no suggestions**, it has no identified + weakness and **shall score 10**. Do not award 8 or 9 "just in case" — a score + below 10 must be justified by at least one finding or suggestion. + +## Analysis Result + +- Provide specific, actionable findings and refer each to a concrete keyword + from the element (e.g. *Vague* — element contains …). +- Only point out findings; do not mention if something is particularly well + defined. +- Categorize findings as major and minor. +- **Suggestions are held to the same bar as findings.** A suggestion is only + warranted when it points at a genuine, actionable weakness in the element not for + stylistic "nice to have" rewordings of an already-clear element. +- Provide an overall score for the element. +- If required, use HTML formatting. diff --git a/validation/ai_checker/guidelines/project/score_architecture_levels.md b/validation/ai_checker/guidelines/project/score_architecture_levels.md new file mode 100644 index 00000000..daaf4a43 --- /dev/null +++ b/validation/ai_checker/guidelines/project/score_architecture_levels.md @@ -0,0 +1,23 @@ + + +# SCORE Architecture Levels + +Project-specific architecture levels for the SCORE Architecture Design Process. +These complement the general architecture guidelines. + +| Level | Scope | Diagram Kinds | +|---|---|---| +| **Feature Architecture** | Integration-level structure and interactions | Component, sequence | +| **Component Architecture** | Component-internal structure | Class, component | +| **Public API** | Externally visible interfaces of a component | Component (public_api) | diff --git a/validation/ai_checker/guidelines/project/score_requirement_levels.md b/validation/ai_checker/guidelines/project/score_requirement_levels.md new file mode 100644 index 00000000..3414a368 --- /dev/null +++ b/validation/ai_checker/guidelines/project/score_requirement_levels.md @@ -0,0 +1,49 @@ + + +# SCORE Requirement Levels + +Project-specific requirement levels for the SCORE Requirements Engineering +Process. These complement the general requirements guidelines. + +| Level | Scope | Derived From | +|---|---|---| +| **Stakeholder Requirement** | Platform-level functionality and safety mechanisms | Standards, customer needs | +| **Feature Requirement** | Integration-level behaviour, independent of component decomposition | Stakeholder Requirements | +| **Component Requirement** | Component-specific implementation details | Feature Requirements | +| **Assumption of Use (AoU)** | Boundary conditions for using a software element (any level) | Safety analyses, architecture | + +## Abstraction Boundaries (what each level must *not* specify) + +Every level is **one step above detailed design**. Detailed design (concrete +error codes/types, message payload layouts, data structures, algorithms, +function signatures, timing constants, …) is **not** a requirement level and is +never demanded by a requirement review. When reviewing, judge completeness only +against the element's own level: + +- **Stakeholder** — describes *what* the platform must offer and which safety + mechanisms apply; does not name features, components, APIs, or mechanisms. +- **Feature** — describes integration-level, externally observable behaviour; + does not prescribe component decomposition, internal interfaces, or + implementation mechanisms (those belong to component requirements / design). +- **Component** — describes a component's externally testable behaviour at + integration-test level; it is **still one level above detailed design**. + Stating that the component "shall report an error", "shall reject the request" + or "shall return a result" is complete — the exact error type/code, payload + structure, or internal algorithm is fixed in detailed design and must **not** + be required here. +- **AoU** — states a boundary condition the using element must satisfy; does not + prescribe how it is met. + +Do **not** lower a score or raise a finding/suggestion because a requirement +omits detail owned by the level(s) below it. diff --git a/validation/ai_checker/guidelines/requirements/requirements_guidelines.md b/validation/ai_checker/guidelines/requirements/requirements_guidelines.md new file mode 100644 index 00000000..e3c7ce77 --- /dev/null +++ b/validation/ai_checker/guidelines/requirements/requirements_guidelines.md @@ -0,0 +1,80 @@ + + +# Requirements Writing Guidelines + +General guidelines for creating and formulating requirements. Project-specific +details (e.g. the available requirement levels) are supplied separately by the +project guidelines. + +## Sentence Template + +Every requirement **shall** follow this structure: + +> **\** shall **\
** **\** **\** **\** + +Of the last three parts (object, parameter, conjunction), at least one is mandatory — the others are optional. + +### Examples + +| Subject | shall | Verb | Object | Parameter | Condition | +|---|---|---|---|---|---| +| The component | shall | detect | if a key-value pair got corrupted | and set its status to INVALID | during every restart of the SW platform. | +| The software platform | shall | enable | users | to ensure the compatibility of application software | across vehicle variants and releases. | +| The linter-tool | shall | check | correctness of .rst files format | | upon each commit. | + +## Quality Criteria + +A well-written requirement is: + +- **Unambiguous** — only one possible interpretation +- **Verifiable** — can be tested or reviewed +- **Atomic** — expresses a single need (one "shall" per requirement) +- **Consistent** — no contradictions with other requirements +- **Complete** — contains subject, verb, and at least one of: object, parameter, or condition +- **Necessary** — traceable to a parent requirement or rationale + +### Avoid + +- Vague terms: *approximately*, *as appropriate*, *user-friendly*, *fast*, *efficient* +- Unbounded lists: *etc.*, *and so on*, *such as* (without closing the list) +- Compound requirements: multiple "shall" statements in one requirement +- Implementation details in stakeholder/feature requirements +- Missing conditions or parameters that leave behaviour undefined + +## Requirement Types Explained + +| Type | Meaning | Verification | +|---|---|---| +| **Functional** | Behaviour that can be observed | Unit/integration test | +| **Interface** | API or protocol specification | Test or inspection | +| **Non-Functional** | Quality attribute (performance, reliability) | Review/analysis | +| **Process** | Process-related constraint | Process review | + +## Pre-flight Checks — Apply Before Raising Any Finding + +1. **Domain terms** — Before flagging a term as *vague*, check whether it has a well-established formal or domain-specific definition (mathematical, safety-engineering, OS/systems, or software-engineering). Formally defined terms (e.g. *monotonic*, *bounded*, *idempotent*, *deterministic*, *ASIL*, …) are precise by definition and shall not be flagged. + +2. **Clarification clauses** — Before flagging a second sentence or clause as a *compound requirement*, check whether it is a clarification, exclusion, negative-scope statement, or rationale that qualifies the main behaviour. Only independent normative *shall* statements and different scopes constitute separate requirements. + +3. **Abstraction patterns** — Before flagging a phrase as a *contradiction*, check whether it describes an abstraction layer. A phrase such as "OS-independent API for OS-native" expresses an abstraction of the underlying mechanism — this is intentional, not contradictory. + +4. **Architectural subjects** — Before flagging a named component in the subject as a *level mismatch*, check whether the requirement constrains an architectural design decision rather than prescribing a code-level implementation detail. Naming an architectural element as the subject does not automatically lower the requirement's level. + +5. **Higher-level parameters** — Before flagging a missing parameter (e.g. list of supported OSes, applicable safety standard) as a *major* finding, consider whether that parameter is resolved at system or project level and intentionally omitted from individual requirements. Downgrade to *minor* unless there is clear evidence it is undefined everywhere. + +6. **Design / implementation constraints** — Before flagging a requirement as *implementation-specific* or *below the expected abstraction level*, check whether it is an intentional design constraint that deliberately restricts the implementation (e.g. mandating a particular transport, mechanism, platform, or safety property, or scoping behaviour to a specific OS such as QNX). Such constraints are legitimate requirements: naming an implementation, technology, or OS context in their subject or object is the *purpose* of the requirement and shall not be flagged as a level mismatch. + +7. **Feature self-reference** — Before flagging a subject as *underspecified* or *ambiguous*, check whether it names the requirement's own feature (e.g. "The communication" in a communication feature, "The logging" in a logging feature). Referring to the feature by name is an acceptable subject and unambiguous in context. Do not raise it as a defect; at most note it as a *minor* consistency suggestion if other requirements use a different wording. + +8. **Detail that belongs to a lower level** — Before flagging a requirement as *underspecified*, *incomplete* or *missing a parameter*, check whether the missing detail belongs to a **lower** abstraction level than the one being reviewed (see the project requirement levels). A requirement is complete when it specifies the behaviour expected *at its own level*; it must **not** pre-empt decisions owned by the level(s) below it. For example, at feature or component level "shall report an error" is complete — the concrete error code, type, message, internal data structure or algorithm is settled in detailed design and shall not be demanded here. Only flag genuinely missing behaviour at the element's own level, not absent lower-level detail. diff --git a/validation/ai_checker/guidelines/requirements_guidelines.md b/validation/ai_checker/guidelines/requirements_guidelines.md deleted file mode 100644 index b2b431a4..00000000 --- a/validation/ai_checker/guidelines/requirements_guidelines.md +++ /dev/null @@ -1,69 +0,0 @@ - - -# Requirements Writing Guidelines - -Guidelines for creating and formulating requirements, derived from the SCORE Requirements Engineering Process. - -## Requirement Levels - -| Level | Scope | Derived From | -|---|---|---| -| **Stakeholder Requirement** | Platform-level functionality and safety mechanisms | Standards, customer needs | -| **Feature Requirement** | Integration-level behaviour, independent of component decomposition | Stakeholder Requirements | -| **Component Requirement** | Component-specific implementation details | Feature Requirements | -| **Assumption of Use (AoU)** | Boundary conditions for using a software element (any level) | Safety analyses, architecture | - -## Sentence Template - -Every requirement **shall** follow this structure: - -> **\** shall **\
** **\** **\** **\** - -Of the last three parts (object, parameter, conjunction), at least one is mandatory — the others are optional. - -### Examples - -| Subject | shall | Verb | Object | Parameter | Condition | -|---|---|---|---|---|---| -| The component | shall | detect | if a key-value pair got corrupted | and set its status to INVALID | during every restart of the SW platform. | -| The software platform | shall | enable | users | to ensure the compatibility of application software | across vehicle variants and releases. | -| The linter-tool | shall | check | correctness of .rst files format | | upon each commit. | - -## Quality Criteria - -A well-written requirement is: - -- **Unambiguous** — only one possible interpretation -- **Verifiable** — can be tested or reviewed -- **Atomic** — expresses a single need (one "shall" per requirement) -- **Consistent** — no contradictions with other requirements -- **Complete** — contains subject, verb, and at least one of: object, parameter, or condition -- **Necessary** — traceable to a parent requirement or rationale - -### Avoid - -- Vague terms: *approximately*, *as appropriate*, *user-friendly*, *fast*, *efficient* -- Unbounded lists: *etc.*, *and so on*, *such as* (without closing the list) -- Compound requirements: multiple "shall" statements in one requirement -- Implementation details in stakeholder/feature requirements -- Missing conditions or parameters that leave behaviour undefined - -## Requirement Types Explained - -| Type | Meaning | Verification | -|---|---|---| -| **Functional** | Behaviour that can be observed | Unit/integration test | -| **Interface** | API or protocol specification | Test or inspection | -| **Non-Functional** | Quality attribute (performance, reliability) | Review/analysis | -| **Process** | Process-related constraint | Process review | diff --git a/validation/ai_checker/requirements.txt b/validation/ai_checker/requirements.txt index 1bca83b6..f8a188ef 100644 --- a/validation/ai_checker/requirements.txt +++ b/validation/ai_checker/requirements.txt @@ -8,143 +8,159 @@ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic -anyio==4.12.1 \ - --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ - --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c +anyio==4.13.0 \ + --hash=sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708 \ + --hash=sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc # via httpx bigtree==1.1.0 \ --hash=sha256:3f1ff63d2d66d31bf19855ddda8884637edf8dd1fc1aa118cf3a750580ece48b \ --hash=sha256:f54f99d842732c91cce39c596a3755a2e8325b1cab5bc6876f5b15bd3942081c # via -r validation/ai_checker/requirements.txt.in -certifi==2026.1.4 \ - --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c \ - --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ + --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d # via # httpcore # httpx # requests -charset-normalizer==3.4.4 \ - --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ - --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ - --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ - --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ - --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ - --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ - --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ - --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ - --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ - --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ - --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ - --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ - --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ - --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ - --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ - --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ - --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ - --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ - --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ - --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ - --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ - --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ - --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ - --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ - --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ - --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ - --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ - --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ - --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ - --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ - --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ - --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ - --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ - --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ - --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ - --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ - --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ - --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ - --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ - --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ - --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ - --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ - --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ - --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ - --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ - --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ - --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ - --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ - --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ - --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ - --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ - --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ - --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ - --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ - --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ - --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ - --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ - --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ - --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ - --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ - --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ - --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ - --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ - --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ - --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ - --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ - --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ - --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ - --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ - --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ - --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ - --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ - --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ - --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ - --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ - --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ - --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ - --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ - --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ - --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ - --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ - --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ - --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ - --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ - --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ - --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ - --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ - --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ - --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ - --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ - --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ - --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ - --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ - --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ - --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ - --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ - --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ - --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ - --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ - --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ - --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ - --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ - --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ - --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ - --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ - --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ - --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ - --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ - --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ - --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ - --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ - --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ - --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via requests -github-copilot-sdk==0.3.0 \ - --hash=sha256:7e241d9b00ebf8bb4d10b2d6101c75fcef38de04d144d729e07fa48394270ee1 \ - --hash=sha256:93b07c46f60cebbbb003d5bddba22eab886849b1d052b98037b52b6434a5bc07 \ - --hash=sha256:b591546d789f9f8243fb59ca71b08cb0bb1dbec818fbef060c3830c6787de2c8 \ - --hash=sha256:c5712d57a2c6291b805c79e039c55c48d858034b1a37fc8e1653925403a028e9 \ - --hash=sha256:ed8f27989158824c754d7febb473bdf25744a1e6bc07a06f114f7e7deebd2c22 \ - --hash=sha256:f4d98a67b8f038885ddd38bd7033d1ac20c3010f04c72ee0fc74ba4984b69ffa +github-copilot-sdk==1.0.1 \ + --hash=sha256:29d850cf5cf2b85c0513b68cba112d4a7aba0a9a447893fc7cc69136dd4e6ac1 \ + --hash=sha256:5f69245b1cb2fc1054e78543ee5c10464b53ebbba11e749c7a42002b02ce6b13 \ + --hash=sha256:a498fffba3d8a521eb73f31d08fef9f534e7c57ae6d0a76fa1848beebb3fb6a9 \ + --hash=sha256:bedb442ee4dd40d43870719f91a9fb6e117ca5d76bbbf1807af80cc77ff64257 \ + --hash=sha256:d1ca7fff62b69f8c9aa24eb2d1f615195730d6b7297050e8e240aa38fbae5eba \ + --hash=sha256:dc4b59dfe034bec6a031b291dbe49de8a8558e42d28aee25213c680ab771d54e # via -r validation/ai_checker/requirements.txt.in h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ @@ -158,112 +174,207 @@ httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via langsmith -idna==3.15 \ - --hash=sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8 \ - --hash=sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 # via # anyio # httpx # requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via -r validation/ai_checker/requirements.txt.in jsonpatch==1.33 \ --hash=sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade \ --hash=sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c # via langchain-core -jsonpointer==3.0.0 \ - --hash=sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 \ - --hash=sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef +jsonpointer==3.1.1 \ + --hash=sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900 \ + --hash=sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca # via jsonpatch -langchain-core==1.4.0 \ - --hash=sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f \ - --hash=sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c +langchain-core==1.4.7 \ + --hash=sha256:7a825d77de0a3f39adbd9d09612a75e85527e14a52c1601089bcc062972d9f2b \ + --hash=sha256:bcadd51951140ecdcba98311dbd931ba5de02a5ba8a2288dad5069c1eea2a13d # via -r validation/ai_checker/requirements.txt.in -langchain-protocol==0.0.15 \ - --hash=sha256:461eb794358f83d5e42635a5797799ffec7b4702314e34edf73ac21e75d3ef79 \ - --hash=sha256:9ab2d11ee73944754f10e037e717098d3a6796f0e58afa9cadda6154e7655ade +langchain-protocol==0.0.17 \ + --hash=sha256:982a08fe152586ed10d4ff3d538c2e0b5766e5f307cdea325e10be3f2c17cae6 \ + --hash=sha256:e7cbe58c205df4b4fd87dc6d5bb23f10e13b236d0e2e1b0b9d05bc2b648f3eea # via langchain-core -langsmith==0.6.3 \ - --hash=sha256:33246769c0bb24e2c17e0c34bb21931084437613cd37faf83bd0978a297b826f \ - --hash=sha256:44fdf8084165513e6bede9dda715e7b460b1b3f57ac69f2ca3f03afa911233ec +langsmith==0.8.16 \ + --hash=sha256:081e57c0175d142192683288740a796eb0eb32d9e703b4bf9133678ceefe3286 \ + --hash=sha256:8c943f0c9185fe2a9637b5b442828b7efd823b1de28d50d14c136c79660f909b # via langchain-core -orjson==3.11.6 \ - --hash=sha256:09dded2de64e77ac0b312ad59f35023548fb87393a57447e1bb36a26c181a90f \ - --hash=sha256:0a54c72259f35299fd033042367df781c2f66d10252955ca1efb7db309b954cb \ - --hash=sha256:0b14dd49f3462b014455a28a4d810d3549bf990567653eb43765cd847df09145 \ - --hash=sha256:132b0ab2e20c73afa85cf142e547511feb3d2f5b7943468984658f3952b467d4 \ - --hash=sha256:150f12e59d6864197770c78126e1a6e07a3da73d1728731bf3bc1e8b96ffdbe6 \ - --hash=sha256:1608999478664de848e5900ce41f25c4ecdfc4beacbc632b6fd55e1a586e5d38 \ - --hash=sha256:1f42da604ee65a6b87eef858c913ce3e5777872b19321d11e6fc6d21de89b64f \ - --hash=sha256:2a42efebc45afabb1448001e90458c4020d5c64fbac8a8dc4045b777db76cb5a \ - --hash=sha256:2a8eeed7d4544cf391a142b0dd06029dac588e96cc692d9ab1c3f05b1e57c7f6 \ - --hash=sha256:2c68de30131481150073d90a5d227a4a421982f42c025ecdfb66157f9579e06f \ - --hash=sha256:2c6b81f47b13dac2caa5d20fbc953c75eb802543abf48403a4703ed3bff225f0 \ - --hash=sha256:300360edf27c8c9bf7047345a94fddf3a8b8922df0ff69d71d854a170cb375cf \ - --hash=sha256:313dfd7184cde50c733fc0d5c8c0e2f09017b573afd11dc36bd7476b30b4cb17 \ - --hash=sha256:314e9c45e0b81b547e3a1cfa3df3e07a815821b3dac9fe8cb75014071d0c16a4 \ - --hash=sha256:351b96b614e3c37a27b8ab048239ebc1e0be76cc17481a430d70a77fb95d3844 \ - --hash=sha256:380f9709c275917af28feb086813923251e11ee10687257cd7f1ea188bcd4485 \ - --hash=sha256:3a63b5e7841ca8635214c6be7c0bf0246aa8c5cd4ef0c419b14362d0b2fb13de \ - --hash=sha256:40dc277999c2ef227dcc13072be879b4cfd325502daeb5c35ed768f706f2bf30 \ - --hash=sha256:46ebee78f709d3ba7a65384cfe285bb0763157c6d2f836e7bde2f12d33a867a2 \ - --hash=sha256:52263949f41b4a4822c6b1353bcc5ee2f7109d53a3b493501d3369d6d0e7937a \ - --hash=sha256:5ae45df804f2d344cffb36c43fdf03c82fb6cd247f5faa41e21891b40dfbf733 \ - --hash=sha256:6026db2692041d2a23fe2545606df591687787825ad5821971ef0974f2c47630 \ - --hash=sha256:6439e742fa7834a24698d358a27346bb203bff356ae0402e7f5df8f749c621a8 \ - --hash=sha256:647d6d034e463764e86670644bdcaf8e68b076e6e74783383b01085ae9ab334f \ - --hash=sha256:65dfa096f4e3a5e02834b681f539a87fbe85adc82001383c0db907557f666bfc \ - --hash=sha256:6dddf9ba706294906c56ef5150a958317b09aa3a8a48df1c52ccf22ec1907eac \ - --hash=sha256:6e0bb2c1ea30ef302f0f89f9bf3e7f9ab5e2af29dc9f80eb87aa99788e4e2d65 \ - --hash=sha256:6f03f30cd8953f75f2a439070c743c7336d10ee940da918d71c6f3556af3ddcf \ - --hash=sha256:71b7cbef8471324966c3738c90ba38775563ef01b512feb5ad4805682188d1b9 \ - --hash=sha256:72c5005eb45bd2535632d4f3bec7ad392832cfc46b62a3021da3b48a67734b45 \ - --hash=sha256:75682d62b1b16b61a30716d7a2ec1f4c36195de4a1c61f6665aedd947b93a5d5 \ - --hash=sha256:7ab85bdbc138e1f73a234db6bb2e4cc1f0fcec8f4bd2bd2430e957a01aadf746 \ - --hash=sha256:825e0a85d189533c6bff7e2fc417a28f6fcea53d27125c4551979aecd6c9a197 \ - --hash=sha256:8523b9cc4ef174ae52414f7699e95ee657c16aa18b3c3c285d48d7966cce9081 \ - --hash=sha256:8d1035d1b25732ec9f971e833a3e299d2b1a330236f75e6fd945ad982c76aaf3 \ - --hash=sha256:8d777ec41a327bd3b7de97ba7bce12cc1007815ca398e4e4de9ec56c022c090b \ - --hash=sha256:905ee036064ff1e1fd1fb800055ac477cdcb547a78c22c1bc2bbf8d5d1a6fb42 \ - --hash=sha256:925e2df51f60aa50f8797830f2adfc05330425803f4105875bb511ced98b7f89 \ - --hash=sha256:931607a8865d21682bb72de54231655c86df1870502d2962dbfd12c82890d077 \ - --hash=sha256:954dae4e080574672a1dfcf2a840eddef0f27bd89b0e94903dd0824e9c1db060 \ - --hash=sha256:955368c11808c89793e847830e1b1007503a5923ddadc108547d3b77df761044 \ - --hash=sha256:9a2d9746a5b5ce20c0908ada451eb56da4ffa01552a50789a0354d8636a02953 \ - --hash=sha256:9d576865a21e5cc6695be8fb78afc812079fd361ce6a027a7d41561b61b33a90 \ - --hash=sha256:a5a5468e5e60f7ef6d7f9044b06c8f94a3c56ba528c6e4f7f06ae95164b595ec \ - --hash=sha256:a613fc37e007143d5b6286dccb1394cd114b07832417006a02b620ddd8279e37 \ - --hash=sha256:a726fa86d2368cd57990f2bd95ef5495a6e613b08fc9585dfe121ec758fb08d1 \ - --hash=sha256:a8173e0d3f6081e7034c51cf984036d02f6bab2a2126de5a759d79f8e5a140e7 \ - --hash=sha256:af44baae65ef386ad971469a8557a0673bb042b0b9fd4397becd9c2dfaa02588 \ - --hash=sha256:afd177f5dd91666d31e9019f1b06d2fcdf8a409a1637ddcb5915085dede85680 \ - --hash=sha256:b04575417a26530637f6ab4b1f7b4f666eb0433491091da4de38611f97f2fcf3 \ - --hash=sha256:b2e2e2456788ca5ea75616c40da06fc885a7dc0389780e8a41bf7c5389ba257b \ - --hash=sha256:b376fb05f20a96ec117d47987dd3b39265c635725bda40661b4c5b73b77b5fde \ - --hash=sha256:b81ffd68f084b4e993e3867acb554a049fa7787cc8710bbcc1e26965580d99be \ - --hash=sha256:b83eb2e40e8c4da6d6b340ee6b1d6125f5195eb1b0ebb7eac23c6d9d4f92d224 \ - --hash=sha256:ba8daee3e999411b50f8b50dbb0a3071dd1845f3f9a1a0a6fa6de86d1689d84d \ - --hash=sha256:c310a48542094e4f7dbb6ac076880994986dda8ca9186a58c3cb70a3514d3231 \ - --hash=sha256:caaed4dad39e271adfadc106fab634d173b2bb23d9cf7e67bd645f879175ebfc \ - --hash=sha256:cbae5c34588dc79938dffb0b6fbe8c531f4dc8a6ad7f39759a9eb5d2da405ef2 \ - --hash=sha256:cded072b9f65fcfd188aead45efa5bd528ba552add619b3ad2a81f67400ec450 \ - --hash=sha256:ce374cb98411356ba906914441fc993f271a7a666d838d8de0e0900dd4a4bc12 \ - --hash=sha256:d8dfa7a5d387f15ecad94cb6b2d2d5f4aeea64efd8d526bfc03c9812d01e1cc0 \ - --hash=sha256:e0ab8d13aa2a3e98b4a43487c9205b2c92c38c054b4237777484d503357c8437 \ - --hash=sha256:e259e85a81d76d9665f03d6129e09e4435531870de5961ddcd0bf6e3a7fde7d7 \ - --hash=sha256:e4ae1670caabb598a88d385798692ce2a1b2f078971b3329cfb85253c6097f5b \ - --hash=sha256:f0f6e9f8ff7905660bc3c8a54cd4a675aa98f7f175cf00a59815e2ff42c0d916 \ - --hash=sha256:f3a135f83185c87c13ff231fcb7dbb2fa4332a376444bd65135b50ff4cc5265c \ - --hash=sha256:f4295948d65ace0a2d8f2c4ccc429668b7eb8af547578ec882e16bf79b0050b2 \ - --hash=sha256:f75c318640acbddc419733b57f8a07515e587a939d8f54363654041fd1f4e465 \ - --hash=sha256:f8515e5910f454fe9a8e13c2bb9dc4bae4c1836313e967e72eb8a4ad874f0248 \ - --hash=sha256:f884c7fb1020d44612bd7ac0db0babba0e2f78b68d9a650c7959bf99c783773f \ - --hash=sha256:f89d104c974eafd7436d7a5fdbc57f7a1e776789959a2f4f1b2eab5c62a339f4 \ - --hash=sha256:f9959c85576beae5cdcaaf39510b15105f1ee8b70d5dacd90152617f57be8c83 \ - --hash=sha256:fe515bb89d59e1e4b48637a964f480b35c0a2676de24e65e55310f6016cca7ce \ - --hash=sha256:fe71f6b283f4f1832204ab8235ce07adad145052614f77c876fcf0dac97bc06f +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +orjson==3.11.9 \ + --hash=sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4 \ + --hash=sha256:03db380e3780fa0015ed776a90f20e8e20bb11dde13b216ce19e5718e3dfba62 \ + --hash=sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979 \ + --hash=sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0 \ + --hash=sha256:0b34789fa0da61cf7bef0546b09c738fb195331e017e477096d129e9105ab03d \ + --hash=sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a \ + --hash=sha256:115ab5f5f4a0f203cc2a5f0fb09aee503a3f771aa08392949ab5ca230c4fbdbd \ + --hash=sha256:135869ef917b8704ea0a94e01620e0c05021c15c52036e4663baffe75e72f8ce \ + --hash=sha256:147302878da387104b66bb4a8b0227d1d487e976ce41a8501916161072ed87b1 \ + --hash=sha256:14ed654580c1ed2bc217352ec82f91b047aef82951aa71c7f64e0dcb03c0e180 \ + --hash=sha256:16969c9d369c98eb084889c6e4d2d39b77c7eb38ceccf8da2a9fff62ae908980 \ + --hash=sha256:19b72ed11572a2ee51a67a903afbe5af504f84ed6f529c0fe44b0ab3fb5cc697 \ + --hash=sha256:231742b4a11dad8d5380a435962c57e91b7c37b79be858f4ef1c0df1a259897e \ + --hash=sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1 \ + --hash=sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09 \ + --hash=sha256:277fefe9d76ee17eb14debf399e3533d4d63b5f677a4d3719eb763536af1f4bd \ + --hash=sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470 \ + --hash=sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b \ + --hash=sha256:33d7d766701847dc6729846362dc27895d2f2d2251264f9d10e7cb9878194877 \ + --hash=sha256:34fd2317602587321faab75ab76c623a0117e80841a6413654f04e47f339a8fb \ + --hash=sha256:3513550321f8c8c811a7c3297b8a630e82dc08e4c10216d07703c997776236cd \ + --hash=sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe \ + --hash=sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97 \ + --hash=sha256:3ebca4179031ee716ed076ffadc29428e900512f6fccee8614c9983157fcf19c \ + --hash=sha256:48ee05097750de0ff69ed5b7bbcf0732182fd57a24043dcc2a1da780a5ead3a5 \ + --hash=sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021 \ + --hash=sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362 \ + --hash=sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206 \ + --hash=sha256:4da3c38a2083ca4aaf9c2a36776cce3e9328e6647b10d118948f3cfb4913ffe4 \ + --hash=sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218 \ + --hash=sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9 \ + --hash=sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f \ + --hash=sha256:53b50b0e14084b8f7e29c5ce84c5af0f1160169b30d8a6914231d97d2fe297d4 \ + --hash=sha256:57ea77fb70a448ce87d18fca050193202a3da5e54598f6501ca5476fb66cfe02 \ + --hash=sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be \ + --hash=sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972 \ + --hash=sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586 \ + --hash=sha256:63e0efbc991250c0b3143488fa57d95affcabbfc63c99c48d625dd37779aafe2 \ + --hash=sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124 \ + --hash=sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa \ + --hash=sha256:71f3db16e69b667b132e0f305a833d5497da302d801508cbb051ed9a9819da47 \ + --hash=sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c \ + --hash=sha256:8697ab6a080a5c46edaad50e2bc5bd8c7ca5c66442d24104fa44ec74910a8244 \ + --hash=sha256:87e4d4ab280b0c87424d47695bec2182caf8cfc17879ea78dab76680194abc13 \ + --hash=sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10 \ + --hash=sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677 \ + --hash=sha256:97d0d932803c1b164fde11cb542a9efcb1e0f63b184537cca65887147906ff48 \ + --hash=sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4 \ + --hash=sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624 \ + --hash=sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49 \ + --hash=sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0 \ + --hash=sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b \ + --hash=sha256:a6082706765a95a6680d812e1daf1c0cfe8adec7831b3ff3b625693f3b461b1c \ + --hash=sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2 \ + --hash=sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db \ + --hash=sha256:ace6c58523302d3b97b6ac5c38a5298a54b473762b6be82726b4265c41029f92 \ + --hash=sha256:b3afcf569c15577a9fe64627292daa3e6b3a70f4fb77a5df246a87ec21681b94 \ + --hash=sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e \ + --hash=sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61 \ + --hash=sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882 \ + --hash=sha256:c5d001196b89fa9cf0a4ab79766cd835b991a166e4b621ba95089edc50c429ff \ + --hash=sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254 \ + --hash=sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f \ + --hash=sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32 \ + --hash=sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff \ + --hash=sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673 \ + --hash=sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291 \ + --hash=sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0 \ + --hash=sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c \ + --hash=sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9 \ + --hash=sha256:f01c4818b3fc9b0da8e096722a84318071eaa118df35f6ed2344da0e73a5444f \ + --hash=sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e \ + --hash=sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7 \ + --hash=sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81 # via langsmith -packaging==25.0 \ - --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ - --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 # via # langchain-core # langsmith @@ -487,9 +598,9 @@ pyyaml==6.0.3 \ # via # -r validation/ai_checker/requirements.txt.in # langchain-core -requests==2.33.0 \ - --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ - --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed # via # langsmith # requests-toolbelt @@ -501,9 +612,9 @@ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -tenacity==9.1.2 \ - --hash=sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb \ - --hash=sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138 +tenacity==9.1.4 \ + --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 \ + --hash=sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a # via langchain-core typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ @@ -523,32 +634,368 @@ urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 # via requests -uuid-utils==0.13.0 \ - --hash=sha256:046cb2756e1597b3de22d24851b769913e192135830486a0a70bf41327f0360c \ - --hash=sha256:3e4f2cc54e6a99c0551158100ead528479ad2596847478cbad624977064ffce3 \ - --hash=sha256:4c17df6427a9e23a4cd7fb9ee1efb53b8abb078660b9bdb2524ca8595022dfe1 \ - --hash=sha256:516adf07f5b2cdb88d50f489c702b5f1a75ae8b2639bfd254f4192d5f7ee261f \ - --hash=sha256:5447a680df6ef8a5a353976aaf4c97cc3a3a22b1ee13671c44227b921e3ae2a9 \ - --hash=sha256:5a88e23e0b2f4203fefe2ccbca5736ee06fcad10e61b5e7e39c8d7904bc13300 \ - --hash=sha256:5dc4c9f749bd2511b8dcbf0891e658d7d86880022963db050722ad7b502b5e22 \ - --hash=sha256:6be6c4d11275f5cc402a4fdba6c2b1ce45fd3d99bb78716cd1cc2cbf6802b2ce \ - --hash=sha256:775347c6110fb71360df17aac74132d8d47c1dbe71233ac98197fc872a791fd2 \ - --hash=sha256:77621cf6ceca7f42173a642a01c01c216f9eaec3b7b65d093d2d6a433ca0a83d \ - --hash=sha256:83628283e977fb212e756bc055df8fdd2f9f589a2e539ba1abe755b8ce8df7a4 \ - --hash=sha256:97985256c2e59b7caa51f5c8515f64d777328562a9c900ec65e9d627baf72737 \ - --hash=sha256:9a5a9eb06c2bb86dd876cd7b2fe927fc8543d14c90d971581db6ffda4a02526f \ - --hash=sha256:aeee3bd89e8de6184a3ab778ce19f5ce9ad32849d1be549516e0ddb257562d8d \ - --hash=sha256:b276b538c57733ed406948584912da422a604313c71479654848b84b9e19c9b0 \ - --hash=sha256:b7ccaa20e24c5f60f41a69ef571ed820737f9b0ade4cbeef56aaa8f80f5aa475 \ - --hash=sha256:bdaf2b77e34b199cf04cde28399495fd1ed951de214a4ece1f3919b2f945bb06 \ - --hash=sha256:c47638ed6334ab19d80f73664f153b04bbb04ab8ce4298d10da6a292d4d21c47 \ - --hash=sha256:cf95f6370ad1a0910ee7b5ad5228fd19c4ae32fe3627389006adaf519408c41e \ - --hash=sha256:e3909a8a1fbd79d7c8bdc874eeb83e23ccb7a7cb0aa821a49596cc96c0cce84b \ - --hash=sha256:e5182e2d95f38e65f2e5bce90648ef56987443da13e145afcd747e584f9bc69c \ - --hash=sha256:eb2f0baf81e82f9769a7684022dca8f3bf801ca1574a3e94df1876e9d6f9271e +uuid-utils==0.16.0 \ + --hash=sha256:04af9966ecd82b78eeba5725e29aa1e86fb8eb84b5443dd6a9935f9fadb6678e \ + --hash=sha256:0681d1bdb7956e0c6d581e7601dabcfb2b08c25d2a65189f4e9b102c94f5ff46 \ + --hash=sha256:06fc7db470c37e5c1ab3fd2cd159697d6f8b279d7d23b5b96bd418b115f8caa9 \ + --hash=sha256:0b3377ce388fd7bf8d231ec9d1d4f58c8e87888ddea93581f60ed6f878a4f722 \ + --hash=sha256:10d21fddb086e69245c4f0f77c7b442471f3a242aa85f62954bff157baa1c5f2 \ + --hash=sha256:10d3c5983f770b1b2847ad811c87a1c9e28f8155d1a27cc581abcd5abb386b64 \ + --hash=sha256:12b6310beb38adc173ec5dc89e98812fd7e3d98f87f3ef01d2ea6ecb5d87994f \ + --hash=sha256:130f7452c1b87b7c16d0bdc1f32a1de531ae4cc4220ed4e691402bbcfc39e0a9 \ + --hash=sha256:13a797e5e8f0dadc18351a5aa013815ddac25dce6864072a539d510910c95f71 \ + --hash=sha256:16dc5c6e439f75b0456114e955983e2156c1f38887733e54d54205d3005223e4 \ + --hash=sha256:1b0dcedf9266bf34a54d5cbe78648eaa627e02352f2a6923ed647530aea2f661 \ + --hash=sha256:1baab8966f9e0097cbaf9cc01ad448b38e616e7b4968ca5e49cb53a74ad91a2f \ + --hash=sha256:1c2df42314b014c9d23330f92887e21d2fc72fde0beb170c7833cd2d22d845a1 \ + --hash=sha256:1c3c5afaaa68b1d6393d653e9fc93a2fde9da1681da01f74b4593f41d31fb5f1 \ + --hash=sha256:207c2a98ca8b065cc93378a3a59744efb88a68e9ecc2c3afefe43d59c864280a \ + --hash=sha256:228701ab6f188b6def24f2add6db64f0794adb1f06d0abacdcec40b0cda13cdf \ + --hash=sha256:22a17e93a371d850ffce8fcdbacc2239f890efe73aa3262b6170c1febc08afe1 \ + --hash=sha256:24e6fa0d0ade7a9ad60a3c296022474983243df5b4e863babb4828a85ef2e52c \ + --hash=sha256:259bab73c241743d684dcc3507feb76f484d720545e4e4805582aeff8e19700b \ + --hash=sha256:26fe23ab60f05de4ad70aaa5b6a4c2a7bbd43055e3dd6f6b31efba0532ac9c71 \ + --hash=sha256:27a071a899ba46a551d6524dbbc5a98b88be176d0f55ddf72cf71c005326ac10 \ + --hash=sha256:2bb3444498e7b099499c8a607d7771377020fa55f7274e46f54106af19f752d7 \ + --hash=sha256:2e2f369dd734050fe96ae4905c58779b09276d47d5e9a0e5cd33ec7982784341 \ + --hash=sha256:38126b353527c5f001e4b24db9e62351eb768d0367febcd68100a4b39a035109 \ + --hash=sha256:39453f1ebf4398fbeb71607f3437e2ac469c9e38b5921755c1e17ad0158a8907 \ + --hash=sha256:3d86ca394e0ea21bdb53784eb99276d263b93d1586f56678cab1414b7ae1d0f3 \ + --hash=sha256:3e1a1f57fe3631e164dad27b24aa81267810e20575f705af3b0fa734f3a21247 \ + --hash=sha256:3ee392fe59808a731b7b6bf4d453fb6e833774921331cceae5f254d1e9c5b97d \ + --hash=sha256:41985e342a30e76366a8becc60bbdb07d72cd1b86ec657b1f31654e9fb1baada \ + --hash=sha256:41a67e546d9adf11c4e4cb5c8e81f000f8b1f000c17912ced089b499855719a5 \ + --hash=sha256:420aa3ca403cedb73490b6ea3aeefeea7e0455f5ce60bbf856390ee872ae3306 \ + --hash=sha256:426a8c9af90242d879706ccf29da56f0b0712e7739fb0bbe16baacabc75596e2 \ + --hash=sha256:4a87a7433b355eadaa200f150da6bb5b87bb6de0adf260883b26cb637aba0410 \ + --hash=sha256:4e35e9a986e86806a61288fac3afbb51317f2580929feefd1661891ffd7b8c24 \ + --hash=sha256:50361aca5c2a770728a6343df85109fe57f89ac026827f34fe0153563cdc9ce7 \ + --hash=sha256:5279bc7ab3c6683f1c67314695bee14d869015acbbc677bdb0015190fe753d16 \ + --hash=sha256:52d2cc8c12a3466cd1727883e0746d8bad5dddd670369eb553ba17fdc3b565ca \ + --hash=sha256:542098f6cb6874aebeff98715f3ab7646fbe0f2ffb24509ca372828c68c4ed0e \ + --hash=sha256:57c3583b1f1c00a94f59726a5e2b988fa209221143919a1af5c2fc24e318fc98 \ + --hash=sha256:57d85f48535dc541060f6b82f277cbcd12b78c04008ccc1039546cfcec027327 \ + --hash=sha256:61a9c4c26ad12ac66fa4bfd0fdb8494724fe7a5b98a9fcd43e78e2b388663dbb \ + --hash=sha256:63bfdf00be51b6b3b79275d6767d034ea5c7a0caa067a35d72861284100cb60a \ + --hash=sha256:66a9c8cedf7695c28e700f6a66bde0809c3b2e0d8a70968be7bfd47c908952e5 \ + --hash=sha256:680799a9ade01d69c53cb9d41392ced24919d4f600bfab5060b61fca37510097 \ + --hash=sha256:6853b627983aa1b4fd95aa52d9e87136eb94a7b3b7de0fbb1db8a498d457eeec \ + --hash=sha256:6da070e75b0e2424728e6f8547647cce36c83f9a6101a08da4849a8ab2b58105 \ + --hash=sha256:7207b25fe534bcf4d57e0110f90670e61c1c38b6f4598ba855af69ab428fc118 \ + --hash=sha256:727fae3f0682191ec9c8ce1cd0f71e81b471a2e26b7c5fd66712fc0f11640aa0 \ + --hash=sha256:733da81d51ea578862d8b9b754e8968b6da2be2b7840aee868917c23cae84015 \ + --hash=sha256:73486b6aa3f755a6c97000f5ea67e7ac78d6df89bf22980789a1e943e24b74f0 \ + --hash=sha256:7525bc59ac4579c32317d2493dd42cf134b9bb50cd0bc6a41dd9f77e4740dde6 \ + --hash=sha256:7555f120a2282d1901c9a632c2398a614101af4fe3f7c8114aa0f1d8c1978855 \ + --hash=sha256:756575d082ea4cb7d2f923d5b640c0efe7c82573aab49220c4e09b62d13737ff \ + --hash=sha256:79824850330e450c7b2fa933572e32192240060937426052fa3fc05134ed3faa \ + --hash=sha256:7f8cf49c05d58523a0f977cb7f11afc05791a0fa164d7303b8365a34750638e7 \ + --hash=sha256:833bc4b3c3fc24be541f67b01b4a75b6b9942a9b7137395b4eb35435948bd6da \ + --hash=sha256:897e8ef0dc5e4ac0b17cf9cae84bb41e560d806280ec5b93db7475b504022105 \ + --hash=sha256:9152bff801ec2ccf630df06d67389090a2c612dea87fbf9a887ab4b222929f6f \ + --hash=sha256:91db59bad97ed2b9d2c6ed25082fe9762b2c422e694fe06786b28cf4e776ac4c \ + --hash=sha256:924a8de04460e4cf65998ad0b6568084f7c51740ebd3254d07a0bcde35a84af6 \ + --hash=sha256:9346ce6eb1fbd8b03a6b331d66016afcb4edcdff6eac708e21391600529a016a \ + --hash=sha256:948485c47d8569a8bf6e86f522a2599fa9134674bee9f483898e601e68c3caca \ + --hash=sha256:95b7f480010ea98a29ee809857a98aa923008c68129af1b39244adccff7377fb \ + --hash=sha256:98e2404713677070cee9a99a1f1e24afd496c18e833ee1b31a0587659452ff80 \ + --hash=sha256:99f8420c3ed59f89a086782ac197e257f4b1debb4545dffa90cf5db23f96c892 \ + --hash=sha256:9a250e111903c4368745fce5ac2aa607bd477c62d3307e45347338fdb64b38e0 \ + --hash=sha256:a0fc6eb3fd821466fbab69cf356c6ec2b7327266bbbc740a2eb57c77c4bef965 \ + --hash=sha256:a49b5a75497643479c919e2e537a4a36224ac3aaa0fada61b75d87024021ac3e \ + --hash=sha256:a4fd5c7936a876ba2606ba124603b559a5c2cea458c59b9c31677e6acc3c53cc \ + --hash=sha256:a632fead2a6505a8df3318d5e95503739b9aa1c518521cd93d83ce00699b78f8 \ + --hash=sha256:a6d3ee32c57898d8415242b08d5dd086bc4f7bcbbb3fc102ef257f3d793eb294 \ + --hash=sha256:a750d8aeb8ae880aa9a2529606bde0e994bcc7448730c953107f357a28e6102e \ + --hash=sha256:aa50261a83991dbb570a00573741455bd8f3249444f7329e5bdcd494799d1504 \ + --hash=sha256:abfbf5e0c47fb31b37164a99515104e449a0bee36a071dc8b105457a2b35a5e6 \ + --hash=sha256:b2e981b1258db444df4cf4bf4c79673570d081d48d35f22d0f86471e0ad795c5 \ + --hash=sha256:b35706350cf9bd4813f1811bebe03cac09795a5a379f90cb3616171f4e9ffc9e \ + --hash=sha256:b42014536943c1a654ff107538c0f7dc39809d8d774ec8dafd19bec05006e568 \ + --hash=sha256:b617a334bb01ef2ff8c22900f5a14125eb9063f602131494cc9dc59519beaa5b \ + --hash=sha256:b8a9a7b1065a12d40f2cc25b7d705ab34954cc57095034367bca39ebcf4a876b \ + --hash=sha256:baf79c8050eb784b252dd34807df73f61130fe8676b61231baccab62530f20ec \ + --hash=sha256:bbb92feb4db08cd76e27b4d3b1a82bfde708447317150c614eb9f761a43b387e \ + --hash=sha256:bece1a6f677ca36047442c465d8166643eed9818b9e43e0bf42d3cf73e92dcff \ + --hash=sha256:c5af79cde16a7600dfccb7d431aec0afd3088ff170b6a09887bf3f7ab3cc7c81 \ + --hash=sha256:c8083284488b84ad178e74add64cfd1e74e8be5e30821e5acbc5019281c658b0 \ + --hash=sha256:c97625e5edfda8b118160ce1e88756f92b1635775f836c168be7bf10928d97fa \ + --hash=sha256:c9f504efeb20ffd9571621658f7c8093c646d33150406d5742e49ff7cd861615 \ + --hash=sha256:caac9c8b1d50e8fbddc76e93bfefbef472978eb45adbfdb6289d578816992953 \ + --hash=sha256:ceef237cf8467fddbf6d8466cc1f6e2c04605ec919046ef5eba10a895b559fcf \ + --hash=sha256:d23fcaf37368a1647319187ef6f8b741bf079f033065899bc2d00a44b0a1214a \ + --hash=sha256:d34cf9681e8892fad2a63e393068e544505408748cd8bf0c3517d753a01528d4 \ + --hash=sha256:d363017a3223de3a57eb6fca135df6ffcef7c534836bff2e71354dce7d10987c \ + --hash=sha256:d5ee0bbbd4ca3968422cd8308f0072520bc73dc760cb26c6fa75ca1aca14d210 \ + --hash=sha256:d6902d4375dfba4c9902c736bb82d3c040417b67f7d0fa48910ddfdb1ac95de7 \ + --hash=sha256:d716e5b35266400d2a2cd349697868179825f113c543e55c9d2ac304991f8d4f \ + --hash=sha256:d89927c47e1a55509e90b7f2fd3e7ff89908c77b61f8f0deda97a89d8854e0f8 \ + --hash=sha256:dc0824a31898ef46a9d84d748c3abe27cdb615ac3773c53cc1f84fc8e66dc7c4 \ + --hash=sha256:de8a365795a76f347f5622621c2bee543cffa0c70949f3ee093bdefc9d926dcc \ + --hash=sha256:e99f9a8b2420b228faba23a637e96efaf5c6a678b2e225870f24431c82707f50 \ + --hash=sha256:ea3265f8e2b452a4870f3298cb1d183dc4e36a3682cbb264dbe46af31267e706 \ + --hash=sha256:ed45fb8732d216426227096b55accbb87cba57febc86a044d90780b090eb99d0 \ + --hash=sha256:efb5252d7c00d586077f10e169d6e6d0b0d0f806d8a085073f0d19b4737aef4e \ + --hash=sha256:f1614572fd9345cdc3dde3f40c237345719fabca1aa87d2d87b321d523cfa34d \ + --hash=sha256:f235ac5827d74ac630cc87f29278cdaa5d2f273613a6e05bbd96df7aa4170776 \ + --hash=sha256:f44b65ae0c329843817d9c90e36a7a3c677b413bf407c99e67db874dac49dad3 \ + --hash=sha256:f7ae4168e1ca0ae69d24207645a8b3cd2b641a0ad15058eda17d2c9898aa89d3 \ + --hash=sha256:fbcac6e6710aa2e4bfbb81762758e01470dc56d5048ba4253acc77c9833568ff # via # langchain-core # langsmith +websockets==16.0 \ + --hash=sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c \ + --hash=sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a \ + --hash=sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe \ + --hash=sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e \ + --hash=sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec \ + --hash=sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1 \ + --hash=sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64 \ + --hash=sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3 \ + --hash=sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8 \ + --hash=sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206 \ + --hash=sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3 \ + --hash=sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156 \ + --hash=sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d \ + --hash=sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9 \ + --hash=sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad \ + --hash=sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2 \ + --hash=sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03 \ + --hash=sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8 \ + --hash=sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230 \ + --hash=sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8 \ + --hash=sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea \ + --hash=sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641 \ + --hash=sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957 \ + --hash=sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6 \ + --hash=sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6 \ + --hash=sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5 \ + --hash=sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f \ + --hash=sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00 \ + --hash=sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e \ + --hash=sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b \ + --hash=sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72 \ + --hash=sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39 \ + --hash=sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9 \ + --hash=sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79 \ + --hash=sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0 \ + --hash=sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac \ + --hash=sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35 \ + --hash=sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0 \ + --hash=sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5 \ + --hash=sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c \ + --hash=sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8 \ + --hash=sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1 \ + --hash=sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244 \ + --hash=sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3 \ + --hash=sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767 \ + --hash=sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a \ + --hash=sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d \ + --hash=sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd \ + --hash=sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e \ + --hash=sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944 \ + --hash=sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82 \ + --hash=sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d \ + --hash=sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4 \ + --hash=sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5 \ + --hash=sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904 \ + --hash=sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde \ + --hash=sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f \ + --hash=sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c \ + --hash=sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89 \ + --hash=sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da \ + --hash=sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4 + # via langsmith +xxhash==3.7.0 \ + --hash=sha256:01cf5c5333aed26cc8d5eea33b8d6398e085e365a704b7372fabdf7ab06441a9 \ + --hash=sha256:030c0fd688fce3569fbb49a2feefd4110cbb0b650186fb4610759ecfac677548 \ + --hash=sha256:03f8ff4474ee61c845758ce00711d7087a770d77efb36f7e74a6e867301000b8 \ + --hash=sha256:040ea63668f9185b92bc74942df09c7e65703deed71431333678fc6e739a9955 \ + --hash=sha256:05ece0fe4d9c9c2728912d1981ae1566cfc83a011571b24732cbf76e1fb70dca \ + --hash=sha256:05fd1254268c59b5cb2a029dfc204275e9fc52de2913f1e53aa8d01442c96b4d \ + --hash=sha256:073c23900a9fbf3d26616c17c830db28af9803677cd5b33aea3224d824111514 \ + --hash=sha256:082c87bfdd2b9f457606c7a4a53457f4c4b48b0cdc48de0277f4349d79bb3d7a \ + --hash=sha256:0c36f89ba026ccc6fde8f48479a2fd9fc450a736cc7c0d5650acfcff8636282e \ + --hash=sha256:0c72fe9c7e3d6dfd7f1e21e224a877917fa09c465694ba4e06464b9511b65544 \ + --hash=sha256:0d23fd49fdc5c8af61fb7104f1ad247954499140f6cb6045b3aa5c99dadbbf28 \ + --hash=sha256:0ff71596bd79816975b3de7130ab1ff4541410285a3c084584eeb1c8239996fd \ + --hash=sha256:1061bc6cec00adf75347b064ee62b220d66d9bc506acaad1418c79eec45a318c \ + --hash=sha256:11dd69b1a34b7b9af29012f390825b0cdb0617c0966560e227ca74daa7478ba9 \ + --hash=sha256:1295325c5a98d552333fa53dc2b026b0ef0ec9c8e73ca3a952990b4c7d65d459 \ + --hash=sha256:12c249621af6d50a05d9f10af894b404157b15819878e18f75fcbb0213a77d07 \ + --hash=sha256:12eca820a5d558633d423bf8bb78ce72a55394823f64089247f788a7e0ae691e \ + --hash=sha256:13805f0461cba0a857924e70ff91ae6d52d2598f79a884e788db80532614a4a1 \ + --hash=sha256:14bf7a54e43825ec131ee7fe3c60e142e7c2c1e676ad0f93fc893432d15414af \ + --hash=sha256:151d7520838d4465461a0b7f4ae488b3b00de16183dd3214c1a6b14bf89d7fb6 \ + --hash=sha256:153c3a4f73563101d4c8102cbff6a5b46f7aa9dbe374eedf1cd3b15fda750566 \ + --hash=sha256:157c49475b34ecea8809e51123d9769a534e139d1247942f7a4bc67710bb2533 \ + --hash=sha256:178959906cb1716a1ce08e0d69c82886c70a15a6f2790fc084fdd146ca30cd49 \ + --hash=sha256:17f8ae90c8e00f225be4899c3023704f23ee6d5638a00c54d6cbe9980068e6f9 \ + --hash=sha256:1910df4756a5ab58cfad8744fc2d0f23926e3efcc346ee76e87b974abab922f4 \ + --hash=sha256:1ad86695c19b1d46fe106925db3c7a37f16be37669dcf58dcc70a9dd6e324676 \ + --hash=sha256:1cc07c639e3a77ef1d32987464d3e408565b8a3be57b545d3542b191054d9923 \ + --hash=sha256:1d398f372496152f1c6933a33566373f8d1b37b98b8c9d608fa6edc0976f23b2 \ + --hash=sha256:2220af08163baf5fa36c2b8af079dc2cbe6e66ae061385267f9472362dfd53c6 \ + --hash=sha256:24cc22070880cc57b830a65cde4e65fa884c6d9b28ae4803b5ee05911e7bafba \ + --hash=sha256:2524a1e20d4c231d13b50f7cf39e44265b055669a64a7a4b9a2a44faa03f19b6 \ + --hash=sha256:2a61e2a3fb23c892496d587b470dee7fa1b58b248a187719c65ea8e94ec13257 \ + --hash=sha256:2d415f18becf6f153046ab6adc97da77e3643a0ee205dae61c4012604113a020 \ + --hash=sha256:31ab1461c77a11461d703c88eb949e132a1c6515933cf675d97ec680f4bd18de \ + --hash=sha256:31e3516a0f829d06ded4a2c0f3c7c5561993256bfa1c493975fb9dc7bfa828a1 \ + --hash=sha256:322b2f0622230f526aeb1738149948a7ae357a9e2ceb1383c6fd1fdaecdafa16 \ + --hash=sha256:3281ba1d1e60ee7a382a7b958513ba03c2c0d5fcbd9a6f7517c0a81251a23422 \ + --hash=sha256:3409b50ddbc76377d938f40a7a4662cd449f743f2c6178fd6162b875bf9b0d4f \ + --hash=sha256:347a93f2b4ce67ce61959665e32a7447c380f8347e55e100daa23766baacf0e5 \ + --hash=sha256:3573a651d146912da9daa9e29e5fbc45994420daaa9ef1e2fa5823e1dc485513 \ + --hash=sha256:363c139bf15e1ac5f136b981d3c077eb551299b1effede7f12faa010b8590a60 \ + --hash=sha256:37d994d0ffe81ef087bb330d392caa809bb5853c77e22ea3f71db024a0543dba \ + --hash=sha256:3afec3a336a2286601a437cb07562ab0227685e6fbb9ec17e8c18457ff348ecf \ + --hash=sha256:3b6b3d28228af044ebcded71c4a3dd86e1dbd7e2f4645bf40f7b5da65bb5fb5a \ + --hash=sha256:3bb5fd680c038fd5229e44e9c493782f90df9bef632fd0499d442374688ff70b \ + --hash=sha256:3beb1de3b1e9694fcdd853e570ee64c631c7062435d2f8c69c1adf809bc086f0 \ + --hash=sha256:3e1860f1e43d40e9d904cf22d93e587ea42e010ebce4160877e46bcab4bc232a \ + --hash=sha256:418a463c3e6a590c0cdc890f8be19adb44a8c8acd175ca5b2a6de77e61d0b386 \ + --hash=sha256:421da671f43a0189b57a4b8be694576308395f92f55ed3badcde67ab95acef81 \ + --hash=sha256:43475925a766d01ca8cd9a857fd87f3d50406983c8506a4c07c4df12adcc867f \ + --hash=sha256:44909f79fb7a4950ec7d96059398f46f634534cd95be9330a3827210af5aaebe \ + --hash=sha256:44fba4a5f1d179b7ddc7b3dc40f56f9209046421679b57025d4d8821b376fd8d \ + --hash=sha256:468f0fc114faaa4b36699f8e328bbc3bb11dc418ba94ac52c26dd736d4b6c637 \ + --hash=sha256:48b542c347c2089f43dc5a6db31d2a6f3cdb04ee33505ec6e9f653834dbb0bde \ + --hash=sha256:496736f86a9bedaf64b0dc70e3539d0766df01c71ea22032698e88f3f04a1ce9 \ + --hash=sha256:49a88183a3e5ab0b69d9bbfc0180cbdb247e8bada19fd9403c538b3aa3c24176 \ + --hash=sha256:49e556558eee5c8c9b2d5da03fd36cfa6c99cae95b3c3887ec64ee1a49ed517a \ + --hash=sha256:4b6d6b33f141158692bd4eafbb96edbc5aa0dabdb593a962db01a91983d4f8fa \ + --hash=sha256:4c2454448ce847c72635827bb75c15c5a3434b03ee1afd28cb6dc6fb2597d830 \ + --hash=sha256:4e15cc9e2817f6481160f930c62842b3ff419e20e13072bcbab12230943092bc \ + --hash=sha256:503722d52a615f2604f5e7611de7d43878df010dc0053094ef91cb9a9ac3d987 \ + --hash=sha256:506a0b488f190f0a06769575e30caf71615c898ed93ab18b0dbcb6dec5c3713c \ + --hash=sha256:50846b9b01f461ee0250d7a701a3d881e9c52ebce335d6e38e0224adc3369f50 \ + --hash=sha256:50e879ebbac351c81565ca108db766d7832f5b8b6a5b14b8c0151f7190028e3d \ + --hash=sha256:54876a4e45101cec2bf8f31a973cda073a23e2e108538dad224ba07f85f22487 \ + --hash=sha256:54a675cb300dda83d71daae2a599389d22db8021a0f8db0dd659e14626eb3ecc \ + --hash=sha256:565df64437a9390f84465dcca33e7377114c7ede8d05cd2cf20081f831ea788e \ + --hash=sha256:5886ad85e9e347911783760a1d16cb6b393e8f9e3b52c982568226cb56927bdc \ + --hash=sha256:5a6ddec83325685e729ca119d1f5c518ec39294212ecd770e60693cdc5f7eb79 \ + --hash=sha256:5b1bde10324f4c31812ae0d0502e92d916ae8917cad7209353f122b8b8f610c3 \ + --hash=sha256:5bf2f1940499839b39fef1561b5ecb6ede9ac34ef4457474e1337fc7ef07c2f3 \ + --hash=sha256:5de686e73690cdaf72b96d4fa083c230ec9020bcc2627ce6316138e2cf2fe2d1 \ + --hash=sha256:5e7ce913b61f35b0c1c839a49ac9c8e75dd8d860150688aed353b0ce1bf409d8 \ + --hash=sha256:5ec1e080a3d02d94ea9335bfab0e3374b877e25411422c18f51a943fa4b46381 \ + --hash=sha256:6318d8b6f6c6c21058928c23289686fc74f37d794170f14b35fecceb515d5e37 \ + --hash=sha256:646a69b56d8145d85f7fd2289d14fba07880c8a5bda406aa256b407481a61f35 \ + --hash=sha256:646b8aa66cf0cec9295dfc4e3ac823ee52e338bada9547f5cf2d674212d04b58 \ + --hash=sha256:6741564a923f082f3c2941c8bb920462ed5b25eaebdd1e161f162233c9a10bc5 \ + --hash=sha256:693d02c6dc7d1aa0a45921d54cd8c1ff629e09dfdc2238471507af1f7a1c6f04 \ + --hash=sha256:6be4d70d9ab76c9f324ead9c01af6ff52c324745ea0c3731682a0cf99720f1fe \ + --hash=sha256:6cc4eefbb542a5d6ffd6d70ea9c502957c925e800f998c5630ecc809d6702bae \ + --hash=sha256:6e83179bbb208fb72774c06ba227d6e410fa3797de33d0d4c00e3935f81da7d2 \ + --hash=sha256:6e934bbae1e0ec74e27d5f0d7f37ef547ce5ff9f0a7e63fb39e559fc99526734 \ + --hash=sha256:6f31143e18e6db136455b16f0e4e6eba943e1889127dd7c649b46a50d54dd836 \ + --hash=sha256:7426ff0dfa76eb47efc2cc59d4a717bfa9dc9938bff5e49e748bca749f6aa616 \ + --hash=sha256:74bbd92f8c7fcc397ba0a11bfdc106bc72ad7f11e3a60277753f87e7532b4d81 \ + --hash=sha256:7553816512c0abb75329c163a1eee77b0802c3757054b910d6e547bd0dbd16b7 \ + --hash=sha256:79f9efdbc828b02c681a7cefc6d4108d63811b20a8fb8518a40cb2c13ed15452 \ + --hash=sha256:7ab9a49c410d8c6c786ab99e79c529938d894c01433130353dd0fe999111077a \ + --hash=sha256:7bd7bc82dd4f185f28f35193c2e968ef46131628e3cac62f639dadf321cba4d1 \ + --hash=sha256:7c4d596b7676f811172687ec567cbafb9e4dea2f9be1bbb4f622410cb7f40f40 \ + --hash=sha256:7c76f18d1268d3dc1c8b8facef5b48a9c6172d4a49113afa2d91745f555c75ff \ + --hash=sha256:7d7148180ec99ba36585b42c8c5de25e9b40191613bc4be68909b4d25a77a852 \ + --hash=sha256:7fbec49f5341bbdea0c471f7d1e2fb41ae8925af9b6f28025c28defd8eb94274 \ + --hash=sha256:84415265192072d8638a3afc3c1bc5995e310570cd9acb54dc46d3939e364fe0 \ + --hash=sha256:845d347df254d6c619f616afa921331bada8614b8d373d58725c663ba97c3605 \ + --hash=sha256:84710b4e449596a6565ab67293858d2d93a54eeec55722d55c8f0a08b6e6de24 \ + --hash=sha256:85f5c0e26d945b5bb475e0a3d95193117498130baa7619357bdc7869c2391b5a \ + --hash=sha256:8653dd7c2eda020545bb2c71c7f7039b53fe7434d0fc1a0a9deb79ab3f1a4fc1 \ + --hash=sha256:875811ba23c543b1a1c3143c926e43996eb27ebb8f52d3500744aa608c275aed \ + --hash=sha256:8c5fcfd806c335bfa2adf1cd0b3110a44fc7b6995c3a648c27489bae85801465 \ + --hash=sha256:8d09dfd2ab135b985daf868b594315ebe11ad86cd9fea46e6c69f19b28f7d25a \ + --hash=sha256:8d4dea659b57443989ef32f4295104fd6912c73d0bf26d1d148bb88a9f159b02 \ + --hash=sha256:8e7edb98dd4721a2694542a35a0bdb989b42892086fd0216f7c48762dfe20844 \ + --hash=sha256:8f4608a06e4d61b7a3425665a46d00e0579122e1a2fae97a0c52953a3aad9aa3 \ + --hash=sha256:8ff00fcc3eb436617ed8556cf15daf76c2b501248361a065625a588af78a0a02 \ + --hash=sha256:90b9d1a8bd37d768ffc92a1f651ec69afc532a96fa1ac2ea7abbed5d630b3237 \ + --hash=sha256:9122ad6f867c4a0f5e655f5c3bdf89103852009dbb442a3d23e688b9e699e800 \ + --hash=sha256:91c3b07cf3362086d8f126c6aecd8e5e9396ad8b2f2219ea7e49a8250c318acd \ + --hash=sha256:921c14e93817842dd0dd9f372890a0f0c72e534650b6ab13c5be5cd0db11d47e \ + --hash=sha256:970f9f8c50961d639cbd0d988c96f80ddf66006de93641719282c4fe7a87c5e6 \ + --hash=sha256:9e6c0d843f1daf85ea23aeb053579135552bde575b7b98af20bfc667b6e4548d \ + --hash=sha256:9f1563fdc8abfc389748e6932c7e4e99c89a53e4ec37d4563c24fc06f5e5644b \ + --hash=sha256:9fd17f14ac0faa12126c2f9ca774a8cf342957265ec3c8669c144e5e6cdb478c \ + --hash=sha256:a04a6cab47e2166435aaf5b9e5ee41d1532cc8300efdef87f2a4d0acb7db19ed \ + --hash=sha256:a169a036bed0995e090d1493b283cc2cc8a6f5046821086b843abefff80643bc \ + --hash=sha256:a2eae53197c6276d5b317f75a1be226bbf440c20b58bf525f36b5d0e1f657ca6 \ + --hash=sha256:a3b19a42111c4057c1547a4a1396a53961dca576a0f6b82bfa88a2d1561764b2 \ + --hash=sha256:a6545e6b409e3d5cbafc850fb84c55a1ca26ed15a6b11e3bf07a0e0cd84517c8 \ + --hash=sha256:a6d73a830b17ef49bc04e00182bd839164c1b3c59c127cd7c54fcb10c7ed8ee8 \ + --hash=sha256:a778b25874cb0f862eaab5986bff4ca49ffb0def7c0a34c237b948b3c6c775b2 \ + --hash=sha256:a7f25baec4c5d851d40718d6fae52285b31683093d4ff5207e63ab306ccf14a5 \ + --hash=sha256:a845a59664d5c531525a467470220f8edc37959e0a6f8e734ffb6654da5c4bee \ + --hash=sha256:a999771ff97bec27d18341be4f3a36b163bb1ac41ec17bef6d2dabd84acd33c7 \ + --hash=sha256:ab9dd2c83c4bbd63e422181a76f13502d049d3ddcac9a1bdc29196263d692bb8 \ + --hash=sha256:abb65b4e947e958f7b3b0d71db3ce447d1bc5f37f5eab871ce7223bda8768a04 \ + --hash=sha256:acbb48679ddf3852c45280c10ff10d52ca2cd1da2e552fb81db1ff786c75d0e4 \ + --hash=sha256:ad37c7792479e49cf96c1ab25517d7003fe0d93687a772ba19a097d235bbe41e \ + --hash=sha256:ad3aa71e12ee634f22b39a0ff439357583706e50765f17f05550f92dbf128a23 \ + --hash=sha256:ae3a39a4d96bdb6f8d154fd7f490c4ad06f0532fcd2bb656052a9a7762cf5d31 \ + --hash=sha256:b081119a6115d2db49e24ab6316b7dcd74651271e9630c7b979999bd0c11973d \ + --hash=sha256:b4e6fe5c6f4e6ad67c1374a7c85c944ca1a8d9672f0a1628201ea5c58e0d4596 \ + --hash=sha256:b59ee2ac81de57771a09ecad09191e840a1d2fae1ef684208320591055768f83 \ + --hash=sha256:b5cd29840505631c6f7dbb8a5d34b742b5e6bbda38fe0b9f54e825f3ea6b61dc \ + --hash=sha256:b7ffeaada9f8699be63d639536b0b60dff73b7d3325b7475c5bc8fdbf4eed47f \ + --hash=sha256:bb16aa13ed175bc9be5c2491ba031b85a9b51c4ed90e0b3d4ebe63cf3fb54f8e \ + --hash=sha256:bfe6f92e3522dcbe8c4281efd74fa7542a336cb00b0e3272c4ec0edabeaeaf67 \ + --hash=sha256:c21625d710f971dd58ae92c5b0c2ca109d2ceba939becc937c5cff9268cd451b \ + --hash=sha256:c3c0059e642b2e7e15c77341a8946f670a403fcd57feecc9e47d68555b9b1c08 \ + --hash=sha256:c40a8ad7d42fe779ac429fe245ed44c54f30e2549173559d70b7167922431701 \ + --hash=sha256:c4fd8acc6e32596350619896feb372033c0920975992d29837c32853bb1feacd \ + --hash=sha256:c50269d0055ac1faecfd559886d2cbe4b730de236585aba0e873f9d9dadbe585 \ + --hash=sha256:c72500a3b6d6c30ebfc135035bcace9eb5884f2dc220804efcaaba43e9f611dd \ + --hash=sha256:c7741c7524961d8c0cb4d4c21b28957ff731a3fd5b5cd8b856dc80a40e9e5acc \ + --hash=sha256:c9b31ab1f28b078a6a1ac1a54eb35e7d5390deddd56870d0be3a0a733d1c321c \ + --hash=sha256:ca12a6d683957a651e3203c1458ff8ab4119aae7363e202e2e820cbfe02df244 \ + --hash=sha256:cb5a888a968b2434abf9ecda357b5d43f10d7b5a6da6fdbbe036208473aff0e2 \ + --hash=sha256:cce1e2782efaf0f595c17fe331cf295882a268c04d5887956e2fc0d262b0fb3a \ + --hash=sha256:cd8ab85c916a58d5c8656ea15e3ce9df836fe2f120a74c296e01d69fab2614b4 \ + --hash=sha256:cee88dfaa6b1b2bfadd3c031fa5f05584870e62fb05dc500942e9900c44fcfda \ + --hash=sha256:cf7424a11a81f59b6f0abdccfbe27c87d552f059ef761471f98245b46b71b5c9 \ + --hash=sha256:d006faf3b491957efcb433489be3c149efe4787b7063d5cddb8ddaefdc60e0c1 \ + --hash=sha256:d1442628c84afa453a9a06a10d74d890d3c1b1e4da313b48b16e1001895fdac4 \ + --hash=sha256:d33fcd60f5546e4b7538a8ae2b2027b51e9905b9a264c32df56de32202997155 \ + --hash=sha256:d41fcda2fa8ca682ebca134a2f2dc02575ba549267585597e73061565795f475 \ + --hash=sha256:d610aa62cdb7d4d497740741772a24a794903bf3e79eaa51d2e800082abe11e5 \ + --hash=sha256:d798c1e291bffb8e37b5bbe0dda77fc767cd19e89cadaf66e6ed5d0ff88c9fe6 \ + --hash=sha256:d7d9110d0c3fb02679972837a033251fd186c529aa62f19c132fc909c74052b8 \ + --hash=sha256:da5b373b1dfce210b8620bdb5d9dae668fe549de67948465dcc39e833d4bbe28 \ + --hash=sha256:dbcd969178d417c2bbd60076f8e407a0e2baf90976eed21c1b818ff8292b902f \ + --hash=sha256:dc026e3b89d98e30a8288c95cb696e77d150b3f0fb7a51f73dcd49ee6b5577fa \ + --hash=sha256:dea2fd4ae84b14aa883ac713faffbb5c26764ec623e00ed34737895be523d1fa \ + --hash=sha256:e64a7c9d7dfca3e0fafcbc5e455519090706a3e36e95d655cec3e04e79f95aaa \ + --hash=sha256:e8ff6ec73110f610425caef3ea875afbfc34caa542f01df3a80f45aadeb9f906 \ + --hash=sha256:ea6daa712f4e094a30830cf01e9b47d03b24d05cc9dab8609f0d9a9db8454712 \ + --hash=sha256:ea85a647fd33d5cf2840027c2e0b7da8868b220d3f05e3866efdda78c440d499 \ + --hash=sha256:ec101643395d7f21405b640f728f6f627e6986557027d740f2f9b220955edafe \ + --hash=sha256:ec68dbba21532c0173a9872298e65c89749f7c9d21538c3a78b5bb6105871568 \ + --hash=sha256:ed4a6efe2dee1655adb73e7ad40c6aa955a6892422b1e3b95de6a34de56e3cbb \ + --hash=sha256:f13319fb8e6ef636f71db3c254d01cbf1543786e10a945a3ff180144618e25b6 \ + --hash=sha256:f14bb8b22a4a91325813e3d553b8963c10cf8c756cff65ee50c194431296c655 \ + --hash=sha256:f1598916cb197681e03e601901e4ab96a9a963de398c59d0964f8a6f44a2b361 \ + --hash=sha256:f1e65d52c2d526734abecb98372c256b7eacce8fdc42e0df8570417fb39e2772 \ + --hash=sha256:f262b8f7599516567e070abf607b9af649052b2c4bd6f9be02b0cb41b7024805 \ + --hash=sha256:f3e7b689c3bce16699efcf736066f5c6cc4472c3840fe4b22bd8279daf4abdac \ + --hash=sha256:f420ad3d41e38194353a498bbc9561fd5a9973a27b536ce46d8583479cf44335 \ + --hash=sha256:f749e52b539e2934171a3718cbf061dc12d74719eddde2d0f025c99637ddbe01 \ + --hash=sha256:f99a15867cbf9fcf753ea72b82a1d6fe6552e6feea3b4842c86a951525685bbb \ + --hash=sha256:f9fd595f1e5941b3d7863e4774e4b30caa6731fc34b9277da032295aa5656ee5 \ + --hash=sha256:fa77e7ec1450d415d20129961814787c9abd9a07f98872f070b1fe96c5084611 \ + --hash=sha256:fc84bf7aa7592f31ec63a3e7b11d624f468a3f19f5238cec7282a42e838ab1d7 \ + --hash=sha256:fd880353cf1ffaf321bc18dd663e111976dbd0d3bbd8a66d58d2b470dfa7f396 \ + --hash=sha256:fdc7d06929ae28dda98297a18eef7b0fd38991a3b405d8d7b55c9ef24c296958 \ + --hash=sha256:fddbbb69a6fff4f421e7a0d1fa28f894b20112e9e3fab306af451e2dfd0e459b \ + --hash=sha256:fe14c356f8b23ad811dc026077a6d4abccdaa7bce5ca98579605550657b6fcfb \ + --hash=sha256:fe32736295ea38e43e7d9424053c8c47c9f64fecfc7c895fb3da9b30b131c9ee \ + --hash=sha256:fe820f104473d1516ecd628993690bc1f79b0e699f32711d42a5a70b3d0f8170 + # via langsmith zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ diff --git a/validation/ai_checker/requirements.txt.in b/validation/ai_checker/requirements.txt.in index 8d143c56..6957205c 100644 --- a/validation/ai_checker/requirements.txt.in +++ b/validation/ai_checker/requirements.txt.in @@ -1,9 +1,10 @@ # Core dependencies for AI checker framework bigtree +jinja2 pydot pydantic pyyaml # LangChain + GitHub Copilot SDK -github-copilot-sdk>=0.3.0 +github-copilot-sdk>=1.0.1 langchain-core>=1.4.0 diff --git a/validation/ai_checker/src/ai_checker/agents/__init__.py b/validation/ai_checker/src/ai_checker/agents/__init__.py new file mode 100644 index 00000000..81f8b009 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/agents/__init__.py @@ -0,0 +1,27 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""AI backend agents that implement ``ai_checker.analysis_agent.AnalysisAgent``. + +Agents are **not** imported eagerly here so each backend can be used without +pulling in the other's dependencies: + +* ``ai_checker.agents.copilot_agent.CopilotAgent`` — default, Copilot SDK, no LangChain. +* ``ai_checker.agents.langchain_agent.LangChainAgent`` — optional, wraps a LangChain + ``BaseChatModel``; needs ``langchain_core``. + +Import the one you need from its submodule. +""" + +from ._errors import CopilotSetupError + +__all__ = ["CopilotSetupError"] diff --git a/validation/ai_checker/src/copilot_adapter/_client_manager.py b/validation/ai_checker/src/ai_checker/agents/_client_manager.py similarity index 75% rename from validation/ai_checker/src/copilot_adapter/_client_manager.py rename to validation/ai_checker/src/ai_checker/agents/_client_manager.py index 60e0c757..6b8e0492 100644 --- a/validation/ai_checker/src/copilot_adapter/_client_manager.py +++ b/validation/ai_checker/src/ai_checker/agents/_client_manager.py @@ -14,10 +14,11 @@ from __future__ import annotations +import asyncio import logging from typing import Any, Optional -from copilot import CopilotClient, SubprocessConfig +from copilot import CopilotClient, RuntimeConnection from ._errors import CopilotSetupError from ._preflight import ( @@ -49,6 +50,7 @@ def __init__(self, copilot_client_options: dict[str, Any] | None = None) -> None self._options: dict[str, Any] = dict(copilot_client_options or {}) self._client: Optional[CopilotClient] = None self._started: bool = False + self._lock: asyncio.Lock = asyncio.Lock() # ------------------------------------------------------------------ # Public interface @@ -58,7 +60,8 @@ async def ensure_client(self) -> CopilotClient: """Return a started, authenticated CopilotClient. Creates and starts the client on the first call; subsequent calls - return the cached instance immediately. + return the cached instance immediately. Thread-safe: concurrent callers + wait on an asyncio.Lock so the CLI subprocess is started exactly once. Pre-flight sequence (runs once, before the CLI is spawned): 1. Resolve the CLI binary path @@ -72,11 +75,12 @@ async def ensure_client(self) -> CopilotClient: CopilotSetupError: With a detailed, actionable message for any failure that prevents the CLI from being used. """ - if self._client is None: - self._client = self._create_client() + async with self._lock: + if self._client is None: + self._client = self._create_client() - if not self._started: - await self._start_and_verify() + if not self._started: + await self._start_and_verify() return self._client @@ -138,28 +142,51 @@ def _create_client(self) -> CopilotClient: ) logger.info("Starting CopilotClient...\n%s", describe_auth_sources()) - _subprocess_fields = frozenset( - { - "cli_path", - "cli_args", - "cwd", - "use_stdio", - "port", - "log_level", - "env", - "github_token", - "use_logged_in_user", - "telemetry", - "session_fs", - "session_idle_timeout_seconds", - } - ) - subprocess_kwargs = {k: v for k, v in opts.items() if k in _subprocess_fields} - return CopilotClient(SubprocessConfig(**subprocess_kwargs)) + + # Map the legacy option keys onto the current SDK client API. + # Transport options (cli_path / cli_url / cli_args / port / use_stdio) + # are folded into a RuntimeConnection; the remaining process-management + # options are passed to CopilotClient directly. + cli_url = opts.get("cli_url") + cli_args = tuple(opts.get("cli_args") or ()) + connection: RuntimeConnection | None + if cli_url: + connection = RuntimeConnection.for_uri(cli_url) + elif opts.get("use_stdio") is False or opts.get("port") is not None: + connection = RuntimeConnection.for_tcp( + port=opts.get("port") or 0, + path=cli_path, + args=cli_args, + ) + elif cli_path: + connection = RuntimeConnection.for_stdio(path=cli_path, args=cli_args) + else: + connection = None + + # Legacy SubprocessConfig key -> current CopilotClient kwarg. + _client_field_map = { + "cwd": "working_directory", + "log_level": "log_level", + "env": "env", + "telemetry": "telemetry", + "session_fs": "session_fs", + "session_idle_timeout_seconds": "session_idle_timeout_seconds", + } + client_kwargs: dict[str, Any] = { + new_key: opts[old_key] + for old_key, new_key in _client_field_map.items() + if old_key in opts + } + if connection is not None: + client_kwargs["connection"] = connection + return CopilotClient(**client_kwargs) async def _start_and_verify(self) -> None: """Start the CLI subprocess and verify authentication.""" - assert self._client is not None + if self._client is None: + raise RuntimeError( + "_start_and_verify() called before the client was created" + ) try: await self._client.start() @@ -180,10 +207,11 @@ async def _start_and_verify(self) -> None: + "\n\n" " Possible fixes:\n" " 1. Run 'copilot' in a terminal and sign in interactively.\n" - " 2. Set COPILOT_GITHUB_TOKEN (or GH_TOKEN / GITHUB_TOKEN)\n" - " and pass it via --action_env=COPILOT_GITHUB_TOKEN.\n" - " 3. Ensure HOME is available in the action environment\n" - " (use_default_shell_env = True in the Bazel rule).\n" + " 2. Export COPILOT_GITHUB_TOKEN (or GH_TOKEN / GITHUB_TOKEN)\n" + " in your shell; the AI test target inherits it via\n" + " RunEnvironmentInfo, so no Bazel flag is required.\n" + " 3. Ensure HOME is set in your shell so the inherited HOME\n" + " lets the CLI read ~/.copilot/config.json.\n" " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" ) from exc raise @@ -209,7 +237,8 @@ async def _verify_auth(self) -> None: LLM call (send_and_wait) will fail with a clear error if auth is truly broken, so we demote this check to a warning-only diagnostic. """ - assert self._client is not None + if self._client is None: + raise RuntimeError("_verify_auth() called before the client was created") try: auth_status = await self._client.get_auth_status() # The SDK uses camelCase on some versions, snake_case on others. diff --git a/validation/ai_checker/src/copilot_adapter/_errors.py b/validation/ai_checker/src/ai_checker/agents/_errors.py similarity index 100% rename from validation/ai_checker/src/copilot_adapter/_errors.py rename to validation/ai_checker/src/ai_checker/agents/_errors.py diff --git a/validation/ai_checker/src/copilot_adapter/_preflight.py b/validation/ai_checker/src/ai_checker/agents/_preflight.py similarity index 92% rename from validation/ai_checker/src/copilot_adapter/_preflight.py rename to validation/ai_checker/src/ai_checker/agents/_preflight.py index cb15e721..12532a53 100644 --- a/validation/ai_checker/src/copilot_adapter/_preflight.py +++ b/validation/ai_checker/src/ai_checker/agents/_preflight.py @@ -106,8 +106,9 @@ def check_auth_sources() -> list[str]: " - $GH_TOKEN\n" " - $GITHUB_TOKEN\n" " - $HOME set so the CLI can read stored OAuth credentials\n" - " Fix: add --action_env=COPILOT_GITHUB_TOKEN to .bazelrc.ai_checker\n" - " and export COPILOT_GITHUB_TOKEN= in your shell.\n" + " Fix: export COPILOT_GITHUB_TOKEN= in your shell before\n" + " running the test. The AI test target inherits it automatically\n" + " (RunEnvironmentInfo), so no Bazel flag is required.\n" " See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md" ] @@ -120,8 +121,10 @@ def describe_auth_sources() -> str: for var in AUTH_ENV_VARS: val = os.environ.get(var) if val: - masked = val[:4] + "..." + val[-4:] if len(val) > 10 else "****" - lines.append(f" [OK] ${var} = {masked}") + # Never echo any portion of a token: report only presence and + # length. These diagnostics are written to the debug log and the + # Bazel test.outputs, which are routinely archived and shared. + lines.append(f" [OK] ${var} is set (length {len(val)})") found_any = True else: lines.append(f" [ ] ${var} — not set") diff --git a/validation/ai_checker/src/ai_checker/agents/copilot_agent.py b/validation/ai_checker/src/ai_checker/agents/copilot_agent.py new file mode 100644 index 00000000..2404e5d5 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/agents/copilot_agent.py @@ -0,0 +1,226 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +Direct GitHub Copilot SDK implementation of :class:`AnalysisAgent`. + +This is the default AI backend. Unlike the optional LangChain adapter +(``LangChainAgent``), it talks to the Copilot SDK directly: it owns a single CLI session per request, +embeds the required JSON schema in the system prompt, and parses the model's +text reply into ``AnalysisResults``. No LangChain dependency is involved. + +The JSON-in-prompt approach (rather than tool calling) is used because the +Copilot CLI's model responds in plain text and ignores tool-calling +instructions — see ``validation/ai_checker/DEVELOPMENT.md``. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +from copilot.session import PermissionHandler +from copilot.session_events import AssistantUsageData + +from ai_checker.analysis_agent import AnalysisAgent, Usage +from ai_checker.analysis_models import AnalysisResults +from ai_checker.constants import DEFAULT_MODEL + +from ._client_manager import CopilotClientManager +from ._errors import CopilotSetupError +from ._preflight import describe_auth_sources + +logger = logging.getLogger(__name__) + + +def _build_json_instruction(schema: type) -> str: + """Build the system-prompt suffix forcing a single JSON object reply.""" + schema_str = json.dumps(schema.model_json_schema(), indent=2) + return ( + "\n\n# CRITICAL OUTPUT FORMAT REQUIREMENT\n" + "You MUST respond with ONLY a valid JSON object. No prose, no markdown, " + "no explanations, no code fences.\n" + "Your ENTIRE response must be a single valid JSON object matching this schema:\n" + f"{schema_str}\n" + "Start your response immediately with `{` and end with `}`." + ) + + +def _extract_json_object(text: str) -> str: + """Return the first balanced top-level ``{...}`` object in ``text``. + + Scans for the first ``{`` and tracks brace depth, ignoring braces inside + double-quoted strings (with escape handling). This is more robust than a + naive ``find('{')`` / ``rfind('}')`` when the model emits trailing prose or + multiple objects after the intended one. + + Raises ``ValueError`` if no balanced object is found. + """ + start = text.find("{") + if start == -1: + raise ValueError("No JSON object found in model response.") + + depth = 0 + in_string = False + escaped = False + for i in range(start, len(text)): + ch = text[i] + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return text[start : i + 1] + + raise ValueError("Unterminated JSON object in model response.") + + +def _parse_results(content: str) -> AnalysisResults: + """Parse the model's text reply into ``AnalysisResults``. + + Extracts the first balanced JSON object (tolerating surrounding prose or + markdown code fences) and validates it against the schema. Raises + ``ValueError`` with the full raw output on any failure. + """ + content = (content or "").strip() + try: + json_text = _extract_json_object(content) + except ValueError as exc: + raise ValueError(f"{exc}\n--- LLM output ---\n{content}\n--- end ---") from exc + try: + parsed = json.loads(json_text) + except json.JSONDecodeError as exc: + raise ValueError( + f"Model returned invalid JSON: {exc}\n" + f"--- LLM output ---\n{content}\n--- end ---" + ) from exc + try: + return AnalysisResults.model_validate(parsed) + except Exception as exc: + raise ValueError( + f"Model output did not match the expected schema: {exc}\n" + f"--- LLM output ---\n{content}\n--- end ---" + ) from exc + + +# Conversion factor from the SDK's nano-AIU billing unit to whole AI credits +# (AIU). copilot_usage.total_nano_aiu is reported in 1e-9 AIU increments. +_NANO_AIU_PER_AIU = 1_000_000_000.0 + + +def _usage_from_event(data: AssistantUsageData) -> Usage: + """Convert one ``assistant.usage`` event into a :class:`Usage`. + + Token counts (``input_tokens`` / ``output_tokens``) and the experimental + ``cost`` (USD) are stable public fields. GitHub Copilot AI credits (AIU) + come from the ``copilot_usage`` billing block, which the SDK marks as an + internal field (``_copilot_usage``) outside its public surface. It is read + defensively via ``getattr`` so a future SDK rename or removal degrades to + zero credits instead of raising — this function is the single isolation + point for that internal dependency. + """ + tokens = (data.input_tokens or 0) + (data.output_tokens or 0) + cost_usd = data.cost or 0.0 + ai_credits = 0.0 + copilot_usage = getattr(data, "_copilot_usage", None) + total_nano_aiu = getattr(copilot_usage, "total_nano_aiu", None) + if total_nano_aiu: + ai_credits = total_nano_aiu / _NANO_AIU_PER_AIU + return Usage(tokens=tokens, cost_usd=cost_usd, ai_credits=ai_credits) + + +class CopilotAgent(AnalysisAgent): + """Default :class:`AnalysisAgent` backed directly by the GitHub Copilot SDK.""" + + def __init__( + self, + model: str = DEFAULT_MODEL, + timeout: float = 120.0, + copilot_client_options: dict[str, Any] | None = None, + ) -> None: + super().__init__() + self.model = model + self.timeout = timeout + self._manager = CopilotClientManager(copilot_client_options or {}) + self._json_instruction = _build_json_instruction(AnalysisResults) + + async def aclose(self) -> None: + """Shut down the underlying Copilot CLI process.""" + await self._manager.close() + + async def analyze(self, system_prompt: str, artefacts_text: str) -> AnalysisResults: + try: + client = await self._manager.ensure_client() + except CopilotSetupError: + raise + except Exception as exc: + raise CopilotSetupError( + f"Unexpected error initialising Copilot SDK: " + f"{type(exc).__name__}: {exc}\n\n" + describe_auth_sources() + ) from exc + + system_content = system_prompt + self._json_instruction + session_config: dict[str, Any] = { + "model": self.model, + "available_tools": [], # Disable built-in tools + "system_message": {"mode": "replace", "content": system_content}, + "infinite_sessions": {"enabled": False}, + } + + session = await client.create_session( + on_permission_request=PermissionHandler.approve_all, + **session_config, + ) + + # Accumulate usage from assistant.usage events emitted during this + # request. send_and_wait only returns the final assistant message, so + # usage (tokens / cost / AI credits) must be collected via the event + # stream. The handler runs synchronously on the same event loop, so the + # in-place add cannot interleave. + request_usage = Usage() + + def _on_event(event: Any) -> None: + nonlocal request_usage + if isinstance(event.data, AssistantUsageData): + request_usage = request_usage + _usage_from_event(event.data) + + unsubscribe = session.on(_on_event) + try: + response = await session.send_and_wait( + artefacts_text, + timeout=self.timeout, + ) + + content = "" + if response and response.data and response.data.content: + content = response.data.content + + if request_usage.is_empty: + logger.debug("Usage not reported by Copilot SDK for this request.") + else: + self._record_usage(request_usage) + + return _parse_results(content) + finally: + unsubscribe() + await session.disconnect() diff --git a/validation/ai_checker/src/ai_checker/agents/langchain_agent.py b/validation/ai_checker/src/ai_checker/agents/langchain_agent.py new file mode 100644 index 00000000..51eccbb1 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/agents/langchain_agent.py @@ -0,0 +1,69 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +LangChain-backed implementation of :class:`AnalysisAgent`. + +This adapter lets any LangChain ``BaseChatModel`` (e.g. ``ChatOpenAI``) serve as +the AI backend. It is **not** on the default path — the default is the direct +``CopilotAgent``. Use this only for SDKs exposed as a LangChain chat model, +supplied via a custom ``create_agent()`` hook. +""" + +from __future__ import annotations + +import logging + +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import HumanMessage, SystemMessage + +from ai_checker.analysis_agent import AnalysisAgent, Usage +from ai_checker.analysis_models import AnalysisResults + +logger = logging.getLogger(__name__) + + +class LangChainAgent(AnalysisAgent): + """Wrap a LangChain ``BaseChatModel`` as an :class:`AnalysisAgent`.""" + + def __init__(self, chat_model: BaseChatModel) -> None: + super().__init__() + self._chat_model = chat_model + self._structured = chat_model.with_structured_output(AnalysisResults) + + async def analyze(self, system_prompt: str, artefacts_text: str) -> AnalysisResults: + response = await self._structured.ainvoke( + [ + SystemMessage(content=system_prompt), + HumanMessage(content=artefacts_text), + ] + ) + + if not hasattr(response, "analyses") or not response.analyses: + raise ValueError( + "AI model returned empty or invalid response. " + f"Expected 'analyses' field, got: {response}" + ) + + # Usage accounting is intentionally not attempted here. ``with_structured_output`` + # returns the parsed pydantic object, not the underlying ``AIMessage``, so the + # per-call ``usage_metadata`` LangChain would otherwise expose is not reachable + # from ``response``. Rather than read attributes that a standard ``BaseChatModel`` + # does not define (which always yielded zero), this adapter reports no usage and + # ``get_usage()`` stays empty. Backends that need usage should use ``CopilotAgent`` + # or supply a custom agent that records it via ``_record_usage``. + logger.debug( + "LangChainAgent does not report usage; structured output omits " + "the underlying message metadata." + ) + + return response diff --git a/validation/ai_checker/src/ai_checker/ai_checker_core.py b/validation/ai_checker/src/ai_checker/ai_checker_core.py index d231b62e..2cb23947 100644 --- a/validation/ai_checker/src/ai_checker/ai_checker_core.py +++ b/validation/ai_checker/src/ai_checker/ai_checker_core.py @@ -24,10 +24,9 @@ import time from typing import Any -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import HumanMessage, SystemMessage from pydantic import ValidationError +from ai_checker.analysis_agent import AnalysisAgent from ai_checker.analysis_cache import AnalysisCache from ai_checker.analysis_models import AnalysisResults from ai_checker.constants import DEFAULT_MODEL @@ -70,8 +69,12 @@ def __init__( self._max_batch_chars = max_batch_chars self._semaphore = asyncio.Semaphore(max_concurrent_requests) - # Set up logger (use fixed name to prevent handler leaks across instances) - self._logger = logging.getLogger(f"{__name__}.AIChecker") + # Set up a per-instance logger. Using id(self) in the name keeps each + # AIChecker's handlers isolated, so two instances with different + # debug_log paths do not clobber each other's file handler (a shared + # fixed name would). One instance is created per CLI run, so the + # logger-registry growth is negligible. + self._logger = logging.getLogger(f"{__name__}.AIChecker.{id(self)}") self._logger.setLevel(logging.DEBUG) self._logger.propagate = False self._logger.handlers.clear() @@ -106,7 +109,11 @@ def _generate_cache_key( """ # Create a deterministic string representation of artefacts artefact_data = json.dumps(artefacts, sort_keys=True) - combined = f"{artefact_data}:{guidelines_content}:{self._model_name}" + # Include the output schema: the agent embeds it in the request (e.g. + # CopilotAgent's JSON instruction), so a schema change must invalidate + # cached results rather than relying on validation to reject them. + schema = json.dumps(AnalysisResults.model_json_schema(), sort_keys=True) + combined = f"{artefact_data}:{guidelines_content}:{self._model_name}:{schema}" return hashlib.sha256(combined.encode("utf-8")).hexdigest() def _format_artefacts_for_analysis( @@ -180,18 +187,19 @@ def _create_batches( async def analyze( self, artefacts: dict[str, dict[str, Any]], - guidelines_content: str, - chat_model: BaseChatModel, + system_prompt: str, + agent: AnalysisAgent, ) -> AnalysisResults: """ - Analyze artefacts using the chat model with structured output. + Analyze artefacts using the agent with structured output. Uses async processing with rate limiting for concurrent requests. Uses caching if enabled to avoid redundant API calls. Args: artefacts: Dictionary mapping artefact IDs to their metadata - guidelines_content: Combined content of all guidelines - chat_model: BaseChatModel instance for AI analysis + system_prompt: Combined system instructions — guidelines plus any + background context — used as the system message + agent: AnalysisAgent instance for AI analysis Returns: AnalysisResults containing structured analyses for each artefact @@ -217,7 +225,7 @@ async def analyze( # Create tasks for all batches to process concurrently batch_tasks = [ - self._analyze_batch_async(i + 1, batch, guidelines_content, chat_model) + self._analyze_batch_async(i + 1, batch, system_prompt, agent) for i, batch in enumerate(batches) ] @@ -227,27 +235,29 @@ async def analyze( # Flatten results from all batches, handling exceptions all_analyses = [] - failed_batches = 0 + failed_batch_numbers = [] for i, batch_results in enumerate(all_batch_results): if isinstance(batch_results, Exception): - failed_batches += 1 - self._logger.warning( - f"--> WARNING: Batch {i + 1} failed with error: " + failed_batch_numbers.append(i + 1) + self._logger.error( + f"--> Batch {i + 1} failed: " f"{type(batch_results).__name__}: {str(batch_results)}" ) else: all_analyses.extend(batch_results) - if failed_batches > 0: - self._logger.warning( - f"--> WARNING: {failed_batches} out of {num_batches} batches failed. " - f"Successfully analyzed {len(all_analyses)} requirement(s)." + if failed_batch_numbers: + failed_list = ", ".join(str(n) for n in failed_batch_numbers) + raise RuntimeError( + f"{len(failed_batch_numbers)} of {num_batches} batches failed " + f"(batch number(s): {failed_list}). " + f"Analyzed {len(all_analyses)} requirement(s) in the " + f"{num_batches - len(failed_batch_numbers)} successful batch(es). " + "See log output above for the per-batch errors." ) # Calculate final statistics - current_total_cost = 0.0 - if chat_model and hasattr(chat_model, "total_costs"): - current_total_cost = getattr(chat_model, "total_costs", 0.0) + usage = agent.get_usage() # Log final statistics total_elapsed = time.time() - total_start_time @@ -255,7 +265,15 @@ async def analyze( average_score = sum(all_scores) / len(all_scores) if all_scores else 0.0 self._logger.info(f"--> Execution time: {total_elapsed:.2f}s") - self._logger.info(f"--> Total costs: ${current_total_cost:.4f} USD") + if usage.is_empty: + self._logger.info("--> Usage not reported by SDK") + else: + if usage.cost_usd: + self._logger.info(f"--> Total cost: ${usage.cost_usd:.4f} USD") + if usage.ai_credits: + self._logger.info(f"--> Total AI credits: {usage.ai_credits:.4f} AIU") + if usage.tokens: + self._logger.info(f"--> Total tokens: {usage.tokens}") self._logger.info(f"--> Overall average score: {average_score:.2f}") return AnalysisResults(analyses=all_analyses) @@ -264,8 +282,8 @@ async def _analyze_batch_async( self, batch_number: int, artefacts: dict[str, dict[str, Any]], - guidelines_content: str, - chat_model: BaseChatModel, + system_prompt: str, + agent: AnalysisAgent, ) -> list[Any]: """ Analyze a batch of artefacts using async with rate limiting. @@ -273,8 +291,8 @@ async def _analyze_batch_async( Args: batch_number: The batch number (1-indexed) for logging artefacts: Dictionary mapping artefact IDs to their metadata - guidelines_content: Combined content of all guidelines - chat_model: BaseChatModel instance for AI analysis + system_prompt: Combined system instructions (guidelines + context) + agent: AnalysisAgent instance for AI analysis Returns: List of analysis results for all artefacts in the batch @@ -288,12 +306,10 @@ async def _analyze_batch_async( self._logger.debug( f"Batch {batch_number} contains artefact IDs: {', '.join(artefacts.keys())}" ) - self._logger.debug( - f"Guidelines content length: {len(guidelines_content)} characters" - ) + self._logger.debug(f"System prompt length: {len(system_prompt)} characters") # Check cache first - cache_hash = self._generate_cache_key(artefacts, guidelines_content) + cache_hash = self._generate_cache_key(artefacts, system_prompt) cached_result = self._cache.get(cache_hash) if cached_result is not None: self._logger.info(f"--> Batch {batch_number}: Completed (from cache)") @@ -302,16 +318,6 @@ async def _analyze_batch_async( # Use semaphore for rate limiting async with self._semaphore: try: - self._logger.debug( - f"Batch {batch_number}: Creating structured chat model..." - ) - - # Create structured chat model - structured_chat = chat_model.with_structured_output(AnalysisResults) - - # Prepare system message with guidelines - system_message = SystemMessage(content=guidelines_content) - # Format requirements for analysis formatted_artefacts = self._format_artefacts_for_analysis(artefacts) @@ -323,13 +329,11 @@ async def _analyze_batch_async( f"Batch {batch_number}: ===== RAW AI MODEL INPUT =====" ) self._logger.debug( - f"Batch {batch_number}: System Message (Guidelines):" + f"Batch {batch_number}: System Prompt (Guidelines + Context):" ) - self._logger.debug(guidelines_content) + self._logger.debug(system_prompt) self._logger.debug(f"Batch {batch_number}: ---") - self._logger.debug( - f"Batch {batch_number}: Human Message (Requirements):" - ) + self._logger.debug(f"Batch {batch_number}: Human Message (Artefacts):") self._logger.debug(formatted_artefacts) self._logger.debug( f"Batch {batch_number}: ===== END RAW AI MODEL INPUT =====" @@ -339,13 +343,9 @@ async def _analyze_batch_async( f"({self._model_name})..." ) - analysis_prompt = HumanMessage(content=formatted_artefacts) - - # Call async invoke + # Call the agent (single structured-output request) start_time = time.time() - response = await structured_chat.ainvoke( - [system_message, analysis_prompt] - ) + response = await agent.analyze(system_prompt, formatted_artefacts) elapsed = time.time() - start_time self._logger.debug( diff --git a/validation/ai_checker/src/ai_checker/analysis_agent.py b/validation/ai_checker/src/ai_checker/analysis_agent.py new file mode 100644 index 00000000..14d339b6 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/analysis_agent.py @@ -0,0 +1,114 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +Agent interface for AI artefact analysis. + +This module defines the thin contract the core analysis loop relies on. An +implementation receives a system prompt (guidelines + optional background +context) and a formatted artefacts string, and returns structured +``AnalysisResults``. + +The interface is intentionally minimal: it decouples the core (batching, +caching, concurrency) from any particular AI SDK. The default implementation +talks to the GitHub Copilot SDK directly (``CopilotAgent``); a LangChain +adapter (``LangChainAgent``) is provided for SDKs exposed as a LangChain +``BaseChatModel``. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from ai_checker.analysis_models import AnalysisResults + + +@dataclass(frozen=True) +class Usage: + """Accumulated AI usage for a run. + + A small, SDK-agnostic value object so the core loop never has to probe + backend-specific response fields. ``ai_credits`` is the GitHub Copilot + billing unit (AIU); ``cost_usd`` is a best-effort dollar figure when the + backend reports one. Any field a backend cannot report stays ``0``. + """ + + tokens: int = 0 + cost_usd: float = 0.0 + ai_credits: float = 0.0 + + def __add__(self, other: "Usage") -> "Usage": + return Usage( + tokens=self.tokens + other.tokens, + cost_usd=self.cost_usd + other.cost_usd, + ai_credits=self.ai_credits + other.ai_credits, + ) + + @property + def is_empty(self) -> bool: + """True when no usage at all was reported.""" + return not (self.tokens or self.cost_usd or self.ai_credits) + + +class AnalysisAgent(ABC): + """Abstract AI backend that produces structured analysis results. + + Implementations should be safe to call concurrently from multiple + coroutines; the core analysis loop fans out batches with + ``asyncio.gather``. + + Usage accounting: implementations that can report AI usage call + :meth:`_record_usage` after each request; the core loop reads the running + total via :meth:`get_usage`. Implementations that cannot report usage + simply never call ``_record_usage`` and ``get_usage`` stays empty. + """ + + def __init__(self) -> None: + # Instance-level usage accumulator. Defined here (not as a class + # attribute) so every agent starts at zero independently and no + # subclass can accidentally share or mutate a class-level value. + self._usage = Usage() + + def get_usage(self) -> Usage: + """Return the usage accumulated across all ``analyze`` calls so far.""" + return self._usage + + def _record_usage(self, usage: Usage) -> None: + """Add ``usage`` to the running total. + + Safe to call from the concurrent ``analyze`` coroutines: the add is a + single synchronous statement with no ``await`` in between, so under + asyncio's cooperative scheduling it cannot interleave. + """ + self._usage = self._usage + usage + + @abstractmethod + async def analyze(self, system_prompt: str, artefacts_text: str) -> AnalysisResults: + """Analyze a batch of artefacts against the system prompt. + + Args: + system_prompt: Combined system instructions — guidelines plus any + background context — used as the system message. + artefacts_text: Formatted artefacts to analyze (the user message). + + Returns: + Structured ``AnalysisResults`` for the artefacts in this batch. + """ + raise NotImplementedError + + async def aclose(self) -> None: + """Release any resources held by the agent (e.g. CLI subprocesses). + + Default is a no-op. Implementations that own external resources must + override this; the orchestrator always calls it after analysis so + cleanup is guaranteed regardless of the concrete backend. + """ + return None diff --git a/validation/ai_checker/src/ai_checker/analysis_cache.py b/validation/ai_checker/src/ai_checker/analysis_cache.py index ef7761b1..c2a79772 100644 --- a/validation/ai_checker/src/ai_checker/analysis_cache.py +++ b/validation/ai_checker/src/ai_checker/analysis_cache.py @@ -22,6 +22,8 @@ import os from typing import Optional +from pydantic import ValidationError + from ai_checker.analysis_models import AnalysisResults logger = logging.getLogger(__name__) @@ -63,9 +65,13 @@ def get(self, cache_hash: str) -> Optional[AnalysisResults]: with open(cache_file, "r", encoding="utf-8") as f: data = json.load(f) return AnalysisResults.model_validate(data) - except Exception as exc: + except (OSError, json.JSONDecodeError, ValidationError) as exc: + # Corrupt/unreadable cache entry — treat as a miss and re-run. + # Narrow to expected I/O and parse/validation errors so an + # unexpected programming error still surfaces instead of being + # silently swallowed as a cache miss. logger.warning( - "Failed to read cache file %s: %s: %s", + "Ignoring unusable cache file %s: %s: %s", cache_file, type(exc).__name__, exc, @@ -88,19 +94,13 @@ def set(self, cache_hash: str, results: AnalysisResults) -> None: try: with open(cache_file, "w", encoding="utf-8") as f: f.write(results.model_dump_json(indent=2)) - except Exception as exc: + except OSError as exc: + # A failed cache write is non-fatal (the result is still returned + # to the caller); log it so a full disk / permission problem is + # visible rather than silently degrading every run to no-cache. logger.warning( "Failed to write cache file %s: %s: %s", cache_file, type(exc).__name__, exc, ) - - def is_enabled(self) -> bool: - """ - Check if caching is enabled. - - Returns: - True if cache directory is configured, False otherwise - """ - return self._cache_dir is not None diff --git a/validation/ai_checker/src/ai_checker/analysis_models.py b/validation/ai_checker/src/ai_checker/analysis_models.py index 486c3c10..60003c71 100644 --- a/validation/ai_checker/src/ai_checker/analysis_models.py +++ b/validation/ai_checker/src/ai_checker/analysis_models.py @@ -31,7 +31,7 @@ class RequirementAnalysis(BaseModel): score: float = Field( description="Numerical score from 0 to 10 representing analysis quality", ge=0.0, - le=10, + le=10.0, ) diff --git a/validation/ai_checker/src/copilot_adapter/__init__.py b/validation/ai_checker/src/ai_checker/extractors/__init__.py similarity index 75% rename from validation/ai_checker/src/copilot_adapter/__init__.py rename to validation/ai_checker/src/ai_checker/extractors/__init__.py index 600db7c0..7a6debba 100644 --- a/validation/ai_checker/src/copilot_adapter/__init__.py +++ b/validation/ai_checker/src/ai_checker/extractors/__init__.py @@ -10,9 +10,4 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Public API for the copilot_adapter package.""" - -from .copilot_langchain import ChatCopilot -from ._errors import CopilotSetupError - -__all__ = ["ChatCopilot", "CopilotSetupError"] +"""Artefact extractors (TRLC requirements, raw PlantUML architecture).""" diff --git a/validation/ai_checker/src/ai_checker/extractors/architecture_extractor.py b/validation/ai_checker/src/ai_checker/extractors/architecture_extractor.py new file mode 100644 index 00000000..37de8f73 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/extractors/architecture_extractor.py @@ -0,0 +1,74 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +Extracts architecture artefacts (PlantUML diagrams) for AI analysis. + +Architecture designs are reviewed from their raw PlantUML source text. The AI +reads the ``.puml`` content directly, so this extractor simply reads each +diagram file and exposes its contents under the standardized artefact format. +""" + +import logging +import os +from typing import Any + +from ai_checker.extractors.base import ArtefactExtractor, unique_key + +logger = logging.getLogger(__name__) + + +class ArchitectureExtractor(ArtefactExtractor): + """Extracts raw PlantUML diagram source for AI analysis.""" + + def __init__(self, puml_files: list[str]): + """ + Initialize the ArchitectureExtractor with diagram file paths. + + Args: + puml_files: List of paths to PlantUML (.puml) source files. + """ + self.puml_files = [os.path.abspath(f) for f in puml_files] + + def extract(self) -> dict[str, dict[str, Any]]: + """ + Read each PlantUML file and return its source as an artefact. + + Returns: + Dictionary mapping diagram names to their metadata: + { + "": { + "type": "plantuml", + "content": "" + } + } + """ + artefacts: dict[str, dict[str, Any]] = {} + for file_path in self.puml_files: + name = os.path.splitext(os.path.basename(file_path))[0] + try: + with open(file_path, encoding="utf-8") as f: + content = f.read() + except OSError as exc: + logger.warning( + "Skipping unreadable PlantUML file %s: %s", file_path, exc + ) + continue + if not content.strip(): + continue + + artefacts[unique_key(artefacts, name)] = { + "type": "plantuml", + "content": content, + } + + return artefacts diff --git a/validation/ai_checker/src/ai_checker/artefact_extractor.py b/validation/ai_checker/src/ai_checker/extractors/base.py similarity index 67% rename from validation/ai_checker/src/ai_checker/artefact_extractor.py rename to validation/ai_checker/src/ai_checker/extractors/base.py index 0194c64e..fea4278a 100644 --- a/validation/ai_checker/src/ai_checker/artefact_extractor.py +++ b/validation/ai_checker/src/ai_checker/extractors/base.py @@ -21,6 +21,27 @@ from typing import Dict, Any +# Safety bound on the disambiguation suffix. Reaching it implies thousands of +# identically-named artefacts, which is a misconfiguration rather than a real +# data set; failing loudly beats an unbounded O(n^2) scan. +_MAX_UNIQUE_KEY_SUFFIX = 10000 + + +def unique_key(existing: dict, base: str) -> str: + """Return ``base`` (or ``base_2``, ``base_3``, …) not already in ``existing``.""" + key = base + suffix = 2 + while key in existing: + if suffix > _MAX_UNIQUE_KEY_SUFFIX: + raise ValueError( + f"Could not derive a unique key for {base!r} after " + f"{_MAX_UNIQUE_KEY_SUFFIX} attempts — too many name collisions." + ) + key = f"{base}_{suffix}" + suffix += 1 + return key + + class ArtefactExtractor(ABC): """ Abstract base class for extracting artefacts for AI analysis. diff --git a/validation/ai_checker/src/ai_checker/requirement_extractor.py b/validation/ai_checker/src/ai_checker/extractors/requirement_extractor.py similarity index 76% rename from validation/ai_checker/src/ai_checker/requirement_extractor.py rename to validation/ai_checker/src/ai_checker/extractors/requirement_extractor.py index 13f56d04..ac1c00db 100644 --- a/validation/ai_checker/src/ai_checker/requirement_extractor.py +++ b/validation/ai_checker/src/ai_checker/extractors/requirement_extractor.py @@ -17,7 +17,7 @@ requirement metadata into a structured format suitable for AI analysis. """ -import argparse +import logging import os from typing import Any @@ -25,7 +25,9 @@ from trlc.errors import Message_Handler from trlc.trlc import Source_Manager -from ai_checker.artefact_extractor import ArtefactExtractor +from ai_checker.extractors.base import ArtefactExtractor, unique_key + +logger = logging.getLogger(__name__) class RequirementExtractor(ArtefactExtractor): @@ -33,7 +35,7 @@ class RequirementExtractor(ArtefactExtractor): def __init__( self, - input_directory: str, + input_directory: str | None = None, dependency_directories: list[str] | None = None, req_files: list[str] | None = None, ): @@ -41,21 +43,28 @@ def __init__( Initialize the RequirementExtractor with directory paths. Args: - input_directory: Path to directory containing TRLC files to - analyze + input_directory: Optional path to a directory containing TRLC + files to analyze. When ``req_files`` is given this is not + required: the explicit files then define both what is parsed + and the grading scope. dependency_directories: Optional list of additional directories for link resolution req_files: Optional list of individual TRLC files to register instead of scanning the entire input directory. When set, - only these files are registered so that other files present - in the same directory (e.g. files not declared in Bazel - srcs) are not picked up by TRLC. + only these files are registered (so other files present in + the same directory are not picked up by TRLC) and exactly + these files form the grading scope. """ - self.input_directory = os.path.abspath(input_directory) + # Use realpath (not abspath) so the scope check in + # extract_requirements_data() is symlink-safe: a symlinked source file + # is compared on its resolved path against the resolved scope. + self.input_directory = ( + os.path.realpath(input_directory) if input_directory else None + ) self.dependency_directories = [ - os.path.abspath(d) for d in (dependency_directories or []) + os.path.realpath(d) for d in (dependency_directories or []) ] - self.req_files = [os.path.abspath(f) for f in (req_files or [])] + self.req_files = [os.path.realpath(f) for f in (req_files or [])] self.symbols: trlc.ast.Symbol_Table | None = None def parse_trlc_files(self) -> trlc.ast.Symbol_Table: @@ -92,7 +101,9 @@ def parse_trlc_files(self) -> trlc.ast.Symbol_Table: else: # Original behaviour: register all directories (input + deps). # Collect all directories and filter out overlapping ones. - all_dirs = [self.input_directory] + self.dependency_directories + all_dirs = ( + [self.input_directory] if self.input_directory else [] + ) + self.dependency_directories # Remove duplicates and filter out directories that are # subdirectories of others @@ -180,8 +191,10 @@ def extract_requirements_data(self) -> list[dict[str, Any]]: """ Extract structured requirement data from TRLC symbol table. - Only extracts requirements from the input_directory, not from - dependency directories. + The grading scope is, in order of precedence, the explicit ``req_files`` + (directory-independent), then the ``input_directory`` prefix, then all + parsed objects. Objects outside the scope (e.g. dependencies) are used + for link resolution only and are not graded. Returns: List of dictionaries, each containing: @@ -195,13 +208,23 @@ def extract_requirements_data(self) -> list[dict[str, Any]]: requirements = [] + # Determine the grading scope. When explicit req files are given they + # define the scope exactly (directory-independent, so requirements + # spread across several directories are all graded). Otherwise fall + # back to the input directory prefix, and if neither is set grade every + # parsed object. + req_file_scope = set(self.req_files) + for obj in self.symbols.iter_record_objects(): - # Only extract requirements from the input directory (not dependencies). - # Use `+ os.sep` to avoid false-positive prefix matches - # (e.g. /foo/bar matching /foo/barbaz). - obj_file_path = os.path.abspath(obj.location.file_name) - if not obj_file_path.startswith(self.input_directory + os.sep): - continue + obj_file_path = os.path.realpath(obj.location.file_name) + if req_file_scope: + if obj_file_path not in req_file_scope: + continue + elif self.input_directory is not None: + # Use `+ os.sep` to avoid false-positive prefix matches + # (e.g. /foo/bar matching /foo/barbaz). + if not obj_file_path.startswith(self.input_directory + os.sep): + continue unique_id = obj.fully_qualified_name() @@ -262,31 +285,21 @@ def extract(self) -> dict[str, dict[str, Any]]: if parent is not None and not isinstance(parent, str): parent = "[not resolved]" - artefacts[req_id] = { + # Guard against duplicate fully-qualified names: disambiguate with a + # suffix instead of letting a later object silently overwrite (and + # drop) an earlier requirement from the graded set. + key = unique_key(artefacts, req_id) + if key != req_id: + logger.warning( + "Duplicate requirement ID %r — grading the duplicate as %r.", + req_id, + key, + ) + + artefacts[key] = { "description": req["description"], "parent": parent, "type": req["requirement_type"], } return artefacts - - -# CLI interface - for direct command-line usage only -def argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input", required=True) - - return parser - - -def main() -> None: - parser = argument_parser() - args = parser.parse_args() - - extractor = RequirementExtractor(args.input) - testfiles = extractor.extract() - print(testfiles) - - -if __name__ == "__main__": - main() diff --git a/validation/ai_checker/src/ai_checker/guidelines_reader.py b/validation/ai_checker/src/ai_checker/guidelines_reader.py index affb4ae8..1c0bf71f 100644 --- a/validation/ai_checker/src/ai_checker/guidelines_reader.py +++ b/validation/ai_checker/src/ai_checker/guidelines_reader.py @@ -11,52 +11,87 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* """ -Reader for guidelines markdown files. +Reader for guidelines and background-context documents. -This module provides functionality to read and manage guidelines -from a directory of markdown files. +This module provides a single reader for text documents used by the AI +checker. It serves two callers that need the same behaviour: + +* **Guidelines** — markdown files loaded from a directory (the graded rules). +* **Background context** — markdown and PlantUML files supplied as an explicit + file list (read-only reference material). + +Both are just "read text files into a name -> content mapping", so they share +one implementation rather than duplicating a near-identical reader. """ import logging import os +from ai_checker.extractors.base import unique_key + logger = logging.getLogger(__name__) class GuidelinesReader: - """Reader for guidelines markdown files.""" - - def __init__(self, guidelines_dir: str): + """Reader for guidelines / context documents. + + Source can be either a directory (scanned for matching files) or an + explicit list of file paths. Files are filtered by extension. + """ + + def __init__( + self, + guidelines_dir: str | None = None, + *, + files: list[str] | None = None, + extensions: tuple[str, ...] = (".md",), + ): """ - Initialize the GuidelinesReader and load all guidelines. + Initialize the reader and load all matching documents. Args: - guidelines_dir: Path to guidelines directory containing - markdown files. + guidelines_dir: Path to a directory to scan (mutually exclusive + with ``files``). Backwards-compatible positional argument. + files: Explicit list of file paths to read. + extensions: Tuple of accepted file extensions (lower-case, with + leading dot). """ self.guidelines_dir = guidelines_dir + self._extensions = extensions - # Dictionary to store all guideline contents keyed by filename - # (without extension) + # Mapping of document name (filename without extension) -> content. self.guidelines: dict[str, str] = {} - # Load all markdown files from the directory - self._load_all_guidelines() + if files is not None: + self._load_files(files) + elif guidelines_dir is not None: + self._load_directory(guidelines_dir) - def _load_all_guidelines(self): - """Load all markdown files from the guidelines directory.""" - if not os.path.isdir(self.guidelines_dir): - logger.warning(f"Guidelines directory not found: {self.guidelines_dir}") + def _matches(self, filename: str) -> bool: + return filename.lower().endswith(self._extensions) + + def _add(self, file_path: str) -> None: + """Read a file and register it under a unique name key.""" + content = self._read_file(file_path) + if not content: + return + base = os.path.splitext(os.path.basename(file_path))[0] + self.guidelines[unique_key(self.guidelines, base)] = content + + def _load_directory(self, directory: str) -> None: + """Load all matching files from a directory.""" + if not os.path.isdir(directory): + logger.warning(f"Guidelines directory not found: {directory}") return + for filename in sorted(os.listdir(directory)): + if self._matches(filename): + self._add(os.path.join(directory, filename)) - for filename in sorted(os.listdir(self.guidelines_dir)): - if filename.endswith(".md"): - file_path = os.path.join(self.guidelines_dir, filename) - # Use filename without extension as key - key = os.path.splitext(filename)[0] - content = self._read_file(file_path) - if content: - self.guidelines[key] = content + def _load_files(self, files: list[str]) -> None: + """Load an explicit list of files, filtered by extension.""" + for file_path in files: + if self._matches(file_path): + self._add(file_path) def _read_file(self, file_path: str) -> str: """Read a file and return its content as a string. @@ -81,20 +116,24 @@ def _read_file(self, file_path: str) -> str: return "" def get_guideline(self, name: str) -> str: - """Get a specific guideline by name. + """Get a specific document by name. Args: - name: Name of the guideline file (without .md extension) + name: Name of the file (without extension) Returns: - Guideline content as string, or empty string if not found + Document content as string, or empty string if not found """ return self.guidelines.get(name, "") def get_all_guidelines(self) -> dict[str, str]: - """Get all guidelines as a dictionary. + """Get all documents as a dictionary. Returns: - Dictionary mapping guideline names to their content + Dictionary mapping document names to their content """ return self.guidelines.copy() + + def get_combined(self) -> str: + """Return all document contents concatenated, in name order.""" + return "\n\n".join(self.guidelines[k] for k in sorted(self.guidelines)) diff --git a/validation/ai_checker/src/ai_checker/orchestrator.py b/validation/ai_checker/src/ai_checker/orchestrator.py index 8d5acc7f..64220bfa 100644 --- a/validation/ai_checker/src/ai_checker/orchestrator.py +++ b/validation/ai_checker/src/ai_checker/orchestrator.py @@ -23,48 +23,70 @@ import logging import os import sys -from typing import Optional - -from langchain_core.language_models.chat_models import BaseChatModel from ai_checker.ai_checker_core import AIChecker +from ai_checker.analysis_agent import AnalysisAgent from ai_checker.analysis_models import AnalysisResults -from ai_checker.requirement_extractor import RequirementExtractor -from ai_checker.result_formatter import ResultFormatter +from ai_checker.extractors.architecture_extractor import ArchitectureExtractor +from ai_checker.extractors.requirement_extractor import RequirementExtractor +from ai_checker.reports.formatter import ResultFormatter from ai_checker.guidelines_reader import GuidelinesReader from ai_checker.constants import DEFAULT_MODEL +# Request timeout heuristic: allow ~1 second of wall-clock per this many +# completion tokens, with a fixed floor. Generous because the Copilot CLI +# round-trip dominates for small requests. +_TOKENS_PER_TIMEOUT_SECOND = 50.0 +_MIN_REQUEST_TIMEOUT_SECONDS = 120.0 + -def _create_default_chat_model( +def _create_default_agent( model_name: str = DEFAULT_MODEL, max_completion_tokens: int = 8192, -) -> BaseChatModel: +) -> AnalysisAgent: """ - Create the default chat model using the GitHub Copilot SDK adapter. - - Uses the ChatCopilot LangChain wrapper as the default AI backend. + Create the default analysis agent backed directly by the Copilot SDK. Args: model_name: Model identifier (e.g. 'gpt-4.1', 'claude-sonnet-4') - max_completion_tokens: Maximum tokens for completion + max_completion_tokens: Maximum tokens for completion (used to size the + request timeout) Returns: - Configured BaseChatModel instance (ChatCopilot) + Configured AnalysisAgent instance (CopilotAgent) """ - from copilot_adapter.copilot_langchain import ChatCopilot + from ai_checker.agents.copilot_agent import CopilotAgent - return ChatCopilot( + return CopilotAgent( model=model_name, - timeout=max(120.0, max_completion_tokens / 50.0), + timeout=max( + _MIN_REQUEST_TIMEOUT_SECONDS, + max_completion_tokens / _TOKENS_PER_TIMEOUT_SECOND, + ), ) +def _agent_from_custom_module(module, model_name: str) -> AnalysisAgent: + """Build an AnalysisAgent from a custom ai_model module. + + The module must expose ``create_agent(model_name) -> AnalysisAgent``. To use + a LangChain model, the function can return + ``LangChainAgent(SomeBaseChatModel(...))``. + """ + if not hasattr(module, "create_agent"): + raise AttributeError( + "Custom ai_model module must define " + "create_agent(model_name) -> AnalysisAgent" + ) + return module.create_agent(model_name=model_name) + + def _load_custom_ai_model_module(custom_path: str): """ Load a custom ai_model module from a file path. - The custom module must provide a `create_chat_model` function with - the signature: `create_chat_model(model_name: str, max_completion_tokens: int) -> BaseChatModel` + The custom module must provide a `create_agent` function with the signature: + `create_agent(model_name: str) -> AnalysisAgent` WARNING: This executes arbitrary Python code from the given path. Only pass paths to files that you own and trust. Never set --custom-ai-model @@ -81,6 +103,10 @@ def _load_custom_ai_model_module(custom_path: str): import importlib.util spec = importlib.util.spec_from_file_location("custom_ai_model", custom_path) + if spec is None or spec.loader is None: + raise ImportError( + f"Could not load a Python module from --custom-ai-model path: {custom_path}" + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module @@ -92,29 +118,50 @@ class AnalysisOrchestrator: extraction and analysis. """ + # TODO(SRP): This class currently owns too many responsibilities — system + # prompt assembly (general + project + context layering), agent + # construction/loading, the event-loop-vs-thread execution strategy in + # analyze_directory(), and report formatting/output. Split into focused + # collaborators (prompt builder, agent factory, async runner, output + # writer) once a unit-test suite exists to make the refactor safe. Tracked + # for a follow-up PR. + def __init__( self, model_name: str = DEFAULT_MODEL, guidelines_path: str = "guidelines", + guideline_files: list[str] | None = None, cache_dir: str | None = None, debug_log: str | None = None, batch_size: int | None = None, custom_ai_model: str | None = None, max_concurrent_requests: int = 5, max_batch_chars: int = 50000, + context_files: list[str] | None = None, + project_guideline_files: list[str] | None = None, ): """ Initialize the orchestrator with AI checker. Args: model_name: Name of the AI model to use - guidelines_path: Relative path to guidelines directory + guidelines_path: Relative path to a guidelines directory (scanned + non-recursively). Used only when ``guideline_files`` is empty. + guideline_files: Optional explicit list of guideline files (.md). + Preferred over ``guidelines_path`` because guideline sets can + span multiple directories; passing the files directly avoids + relying on a single directory scan. cache_dir: Optional directory path for caching results debug_log: Optional file path for detailed debug logging batch_size: Optional number of requirements to process per batch custom_ai_model: Optional path to custom ai_model.py file max_concurrent_requests: Maximum number of concurrent API requests max_batch_chars: Maximum total characters per batch + context_files: Optional list of background-context files (.md/.puml) + injected into the system prompt as read-only reference material. + project_guideline_files: Optional list of project-specific guideline + files (.md) injected into the system prompt as graded rules, + layered on top of the general and type guidelines. """ self.model_name = model_name self.guidelines_path = guidelines_path @@ -123,25 +170,61 @@ def __init__( # Initialize requirement extractor (no input directory yet) self.requirement_extractor = None - # Load guidelines using GuidelinesReader - self.guidelines_reader = GuidelinesReader(guidelines_path) - all_guidelines = self.guidelines_reader.get_all_guidelines() - self.guidelines_content = "\n\n".join(all_guidelines.values()) - - # Create AI model (private member) - self._chat_model: BaseChatModel = None - if custom_ai_model and os.path.exists(custom_ai_model): - # Use custom ai_model.py provided by the user + # Load guidelines using GuidelinesReader. Prefer an explicit file list + # (handles guideline sets that span multiple directories) and fall back + # to scanning a single directory for direct CLI use. + if guideline_files: + self.guidelines_reader = GuidelinesReader(files=guideline_files) + else: + self.guidelines_reader = GuidelinesReader(guidelines_path) + self.guidelines_content = self.guidelines_reader.get_combined() + + self.system_prompt = self.guidelines_content + + # Layer project-specific guidelines (graded) on top of the general and + # type guidelines. These carry project details such as requirement or + # architecture levels and are evaluated like any other guideline. + if project_guideline_files: + project_reader = GuidelinesReader(files=project_guideline_files) + project_content = project_reader.get_combined() + if project_content: + self.system_prompt += ( + "\n\n# PROJECT-SPECIFIC GUIDELINES\n\n" + project_content + ) + + # Load optional background context (markdown + plantuml) and append it + # as a clearly labelled, read-only section of the system prompt. + if context_files: + context_reader = GuidelinesReader( + files=context_files, extensions=(".md", ".puml") + ) + context_content = context_reader.get_combined() + if context_content: + self.system_prompt += ( + "\n\n# BACKGROUND CONTEXT (reference only — not graded)\n\n" + + context_content + ) + + # Create the analysis agent (private member) + logger = logging.getLogger(__name__) + if custom_ai_model: + # SECURITY: _load_custom_ai_model_module() executes arbitrary Python + # from this path. The path comes from the trusted --custom-ai-model + # flag / Bazel target; never wire it to untrusted external input. A + # set-but-missing path is a hard error rather than a silent fallback + # to the default agent, so a typo cannot mask the intended backend. + if not os.path.exists(custom_ai_model): + raise FileNotFoundError( + f"--custom-ai-model path does not exist: {custom_ai_model}" + ) ai_model_module = _load_custom_ai_model_module(custom_ai_model) - self._chat_model = ai_model_module.create_chat_model( - model_name=model_name, - max_completion_tokens=8192, + self._agent: AnalysisAgent = _agent_from_custom_module( + ai_model_module, model_name ) else: - # Default: use GitHub Copilot SDK via ChatCopilot adapter - logger = logging.getLogger(__name__) - logger.info("--> Using default ChatCopilot model adapter") - self._chat_model = _create_default_chat_model( + # Default: use the GitHub Copilot SDK directly via CopilotAgent + logger.info("--> Using default CopilotAgent (Copilot SDK)") + self._agent = _create_default_agent( model_name=model_name, max_completion_tokens=8192, ) @@ -165,54 +248,85 @@ def __init__( # Stored artefacts from extraction (reused for formatting) self._extracted_artefacts = None + # Guard: agent is closed after the first analyze_directory() call. + self._agent_closed: bool = False + def analyze_directory( self, - input_dir: str, + input_dir: str | None = None, dependency_dirs: list[str] | None = None, req_files: list[str] | None = None, + artefact_type: str = "requirements", + puml_files: list[str] | None = None, ) -> AnalysisResults: """ - Extract and analyze artefacts from a directory using TRLC - extractor. + Extract and analyze artefacts using the extractor for ``artefact_type``. Args: input_dir: Path to directory containing files to analyze dependency_dirs: Optional list of directories containing - dependencies for link resolution + dependencies for link resolution (requirements only) req_files: Optional list of individual TRLC files to register - instead of scanning the entire input directory. When set, - only these files are parsed so that unreferenced files - present in the same directory are not picked up. + instead of scanning the entire input directory (requirements + only). When set, only those files are parsed. + artefact_type: Either ``"requirements"`` (TRLC) or + ``"architecture"`` (raw PlantUML). + puml_files: List of PlantUML files to analyze (architecture only). Returns: AnalysisResults containing structured analyses for each artefact """ - # Initialize TRLC requirement extractor - self.artefact_extractor = RequirementExtractor( - input_dir, - dependency_dirs, - req_files=req_files or [], - ) + if self._agent_closed: + raise RuntimeError( + "AnalysisOrchestrator.analyze_directory() may only be called once. " + "Create a new orchestrator instance for each analysis run." + ) + + # Remember the artefact type for report metadata. + self._artefact_type = artefact_type + + # Select the extractor for the requested artefact type. + if artefact_type == "architecture": + self.artefact_extractor = ArchitectureExtractor(puml_files or []) + else: + self.artefact_extractor = RequirementExtractor( + input_dir, + dependency_dirs, + req_files=req_files or [], + ) # Extract artefacts artefacts = self.artefact_extractor.extract() self._extracted_artefacts = artefacts if not artefacts: - print( - f"WARNING: No artefacts found in '{input_dir}'. " - "Architecture analysis is not yet implemented.", - file=sys.stderr, + logging.getLogger(__name__).warning( + "No '%s' artefacts found in '%s'.", + artefact_type, + input_dir or "", ) return AnalysisResults(analyses=[]) - # Analyze artefacts using AI checker with guidelines and chat model. + # Analyze artefacts using AI checker with the assembled system prompt + # and agent. Close the agent afterward so the CLI subprocess is shut + # down in the same event loop that started it. # asyncio.run() will raise RuntimeError if there is already a running # event loop (e.g. inside pytest-asyncio or Jupyter). In that case, # delegate to a fresh thread that owns its own event loop. - coro = self.ai_checker.analyze( - artefacts, self.guidelines_content, self._chat_model - ) + agent = self._agent + + async def _analyze_and_close() -> AnalysisResults: + try: + return await self.ai_checker.analyze( + artefacts, self.system_prompt, agent + ) + finally: + # aclose() is part of the AnalysisAgent interface (no-op by + # default), so cleanup is guaranteed for every backend. + await agent.aclose() + self._agent_closed = True + + coro = _analyze_and_close() try: asyncio.get_running_loop() # We're inside a running loop — run the coroutine in a new thread. @@ -227,17 +341,22 @@ def analyze_directory( def format_and_output( self, analysis_results: AnalysisResults, - output_file: str = None, - html_file: str = None, - guidelines_output_dir: str = None, + output_file: str | None = None, + html_file: str | None = None, + guidelines_output_dir: str | None = None, + rst_file: str | None = None, ) -> None: """Format and output analysis results. + Builds one report in memory and renders each requested format directly + from it. + Args: analysis_results: AnalysisResults to format and output output_file: Output file for JSON results (None for stdout) html_file: Output file for HTML report (optional) guidelines_output_dir: Output directory for guideline pages (optional) + rst_file: Output file for reStructuredText report (optional) """ # Use previously extracted artefacts (avoids re-parsing) original_artefacts = self._extracted_artefacts @@ -249,6 +368,7 @@ def format_and_output( guidelines_reader=self.guidelines_reader, guidelines_output_dir=guidelines_output_dir, original_requirements=original_artefacts, + artefact_type=getattr(self, "_artefact_type", "requirements"), ) # Output JSON results (primary output) @@ -261,6 +381,10 @@ def format_and_output( if html_file: self.result_formatter.output(html_file) + # Output reStructuredText report if requested + if rst_file: + self.result_formatter.output(rst_file) + def argument_parser() -> argparse.ArgumentParser: """Create argument parser for CLI.""" @@ -282,8 +406,55 @@ def argument_parser() -> argparse.ArgumentParser: parser.add_argument( "-i", "--input", - required=True, - help="Path to directory containing TRLC files to analyze", + default=None, + help=( + "Path to directory containing artefact files to analyze. Optional " + "when --req-file is used (the req files then define the grading " + "scope) or for architecture review (which uses --puml-file)." + ), + ) + parser.add_argument( + "--artefact-type", + choices=["requirements", "architecture"], + default="requirements", + dest="artefact_type", + help=( + "Type of artefacts to analyze: 'requirements' (TRLC) or " + "'architecture' (raw PlantUML). Default: requirements." + ), + ) + parser.add_argument( + "--puml-file", + action="append", + default=[], + dest="puml_file", + help=( + "Individual PlantUML file to analyze for architecture review " + "(can be specified multiple times). Used with " + "--artefact-type architecture." + ), + ) + parser.add_argument( + "--context-file", + action="append", + default=[], + dest="context_file", + help=( + "Background-context file (.md or .puml) injected into the system " + "prompt as read-only reference material (can be specified multiple " + "times)." + ), + ) + parser.add_argument( + "--project-guidelines", + action="append", + default=[], + dest="project_guidelines", + help=( + "Project-specific guideline file (.md) injected into the system " + "prompt as a graded rule, layered on top of the general and type " + "guidelines (can be specified multiple times)." + ), ) parser.add_argument( "--deps", @@ -305,12 +476,29 @@ def argument_parser() -> argparse.ArgumentParser: default=None, help="Output file for HTML report (optional)", ) + parser.add_argument( + "--rst", + default=None, + help="Output file for reStructuredText report (optional)", + ) parser.add_argument( "-g", "--guidelines", default="guidelines", help="Relative path to guidelines directory (default: guidelines)", ) + parser.add_argument( + "--guidelines-file", + action="append", + default=[], + dest="guidelines_file", + help=( + "Explicit guideline file (.md) to load (can be specified multiple " + "times). Preferred over --guidelines: guideline sets can span " + "several directories, and the Bazel rules pass every guideline " + "file directly so none are dropped by a single directory scan." + ), + ) parser.add_argument( "-m", "--model", @@ -365,42 +553,121 @@ def argument_parser() -> argparse.ArgumentParser: action="store_true", help="Enable verbose debug logging to debug log file (requires --debug-log)", ) + parser.add_argument( + "--score-threshold", + type=float, + default=None, + dest="score_threshold", + help=( + "Minimum average score (0-10) required to succeed. When set, the " + "process exits non-zero if the average score is below this value. " + "Used by the Bazel test rules." + ), + ) return parser +def _report_location(output_path: str | None) -> str | None: + """Return a human-friendly location for the generated reports. + + Inside a Bazel test, Bazel copies the undeclared outputs to + ``bazel-testlogs///test.outputs/`` — a stable path relative + to the workspace root. Derive it from ``$TEST_TARGET`` (``//package:name``) + so the message points at the committed tree rather than the sandbox temp + directory. Outside a test, fall back to the report's own directory. + """ + test_target = os.environ.get("TEST_TARGET") + if test_target and test_target.startswith("//"): + package, _, name = test_target[2:].partition(":") + return f"bazel-testlogs/{package}/{name}/test.outputs" + if output_path: + return os.path.dirname(os.path.abspath(output_path)) + return None + + def main() -> None: """Main entry point for the orchestrator CLI.""" parser = argument_parser() args = parser.parse_args() - try: - # Initialize orchestrator and analyze - orchestrator = AnalysisOrchestrator( - model_name=args.model, - guidelines_path=args.guidelines, - cache_dir=args.cache, - debug_log=args.debug_log if args.verbose else None, - batch_size=args.batch_size, - custom_ai_model=args.custom_ai_model, - max_concurrent_requests=args.max_concurrent_requests, - max_batch_chars=args.max_batch_chars, - ) - analysis_results = orchestrator.analyze_directory( - args.input, - args.deps, - req_files=args.req_file or None, - ) + # When run as a Bazel test, write every report into the test's + # undeclared-outputs directory automatically. This is the native test + # "artifact" mechanism: unlike a build action a test cannot declare output + # files, but the runner exposes $TEST_UNDECLARED_OUTPUTS_DIR and Bazel zips + # its contents into bazel-testlogs/.../test.outputs/outputs.zip. Detecting + # it here keeps the Bazel test launcher trivial (no output-path plumbing). + test_output_dir = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") or os.environ.get( + "TEST_TMPDIR" + ) + if test_output_dir: + os.makedirs(test_output_dir, exist_ok=True) + defaults = { + "output": "analysis.json", + "html": "analysis.html", + "rst": "analysis.rst", + "guidelines_output": "guidelines", + "debug_log": "debug.log", + } + for attr, filename in defaults.items(): + if getattr(args, attr) is None: + setattr(args, attr, os.path.join(test_output_dir, filename)) + + orchestrator = AnalysisOrchestrator( + model_name=args.model, + guidelines_path=args.guidelines, + guideline_files=args.guidelines_file or None, + cache_dir=args.cache, + debug_log=args.debug_log if args.verbose else None, + batch_size=args.batch_size, + custom_ai_model=args.custom_ai_model, + max_concurrent_requests=args.max_concurrent_requests, + max_batch_chars=args.max_batch_chars, + context_files=args.context_file or None, + project_guideline_files=args.project_guidelines or None, + ) + analysis_results = orchestrator.analyze_directory( + args.input, + args.deps, + req_files=args.req_file or None, + artefact_type=args.artefact_type, + puml_files=args.puml_file or None, + ) - # Format and output results - orchestrator.format_and_output( - analysis_results, - output_file=args.output, - html_file=args.html, - guidelines_output_dir=args.guidelines_output, + # Format and output results + orchestrator.format_and_output( + analysis_results, + output_file=args.output, + html_file=args.html, + guidelines_output_dir=args.guidelines_output, + rst_file=args.rst, + ) + + # Tell the user where to find the reports. Inside a Bazel test the reports + # live under bazel-testlogs///test.outputs/ — a stable, + # workspace-root-relative path that is far more useful than the sandbox + # temp dir or the test launcher script. Derive it from $TEST_TARGET + # (//package:name), which Bazel sets for every test. + report_location = _report_location(args.output) + if report_location: + print(f"AI analysis reports: {report_location}") + + # Enforce the score threshold when requested (Bazel test rules). The + # analysis itself runs as the test action, so a failing score fails the + # test directly. + if args.score_threshold is not None: + scores = [a.score for a in analysis_results.analyses] + average = sum(scores) / len(scores) if scores else 0.0 + if average < args.score_threshold: + print( + f"ERROR: Average score {average:.2f} is below threshold " + f"{args.score_threshold:.2f}", + file=sys.stderr, + ) + sys.exit(1) + print( + f"AI analysis complete. Average score: {average:.2f} " + f"(threshold: {args.score_threshold:.2f})" ) - except Exception: - # Let exceptions propagate with full traceback - raise if __name__ == "__main__": diff --git a/validation/ai_checker/src/ai_checker/reports/__init__.py b/validation/ai_checker/src/ai_checker/reports/__init__.py new file mode 100644 index 00000000..93884904 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/__init__.py @@ -0,0 +1,13 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Report models and renderers (JSON / HTML / reST) for analysis results.""" diff --git a/validation/ai_checker/src/ai_checker/reports/base.py b/validation/ai_checker/src/ai_checker/reports/base.py new file mode 100644 index 00000000..81f1c4ed --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/base.py @@ -0,0 +1,48 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Renderer interface for analysis reports.""" + +from abc import ABC, abstractmethod + +from ai_checker.reports.models import AnalysisReport + + +class ReportRenderer(ABC): + """Render an :class:`AnalysisReport` to a single output format. + + Implementations are pure functions of the report — no external state — so + the same report object can be rendered to any format. + """ + + #: File extension this renderer produces (including the leading dot). + extension: str = "" + + def __init__(self, guidelines_output_dir: str | None = None) -> None: + # Optional directory for companion guideline pages; ignored by + # renderers that emit a self-contained document (e.g. JSON). + self._guidelines_output_dir = guidelines_output_dir + # Set by the caller before render() when relative links to companion + # files must be computed from the final output location. + self._out_path: str | None = None + + @abstractmethod + def render(self, report: AnalysisReport) -> str: + """Return the rendered document as a string.""" + raise NotImplementedError + + def write_extras(self, report: AnalysisReport, out_path: str) -> None: + """Write any companion files next to ``out_path`` (e.g. guideline pages). + + Default: no extras. + """ + return None diff --git a/validation/ai_checker/src/ai_checker/reports/formatter.py b/validation/ai_checker/src/ai_checker/reports/formatter.py new file mode 100644 index 00000000..ec02af67 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/formatter.py @@ -0,0 +1,152 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +Facade that builds an :class:`AnalysisReport` and renders it. + +It assembles **one** report object in memory from the analysis results plus +metadata and guideline texts, then renders the requested format directly from +that object. JSON is just one of the renderers — HTML / reST are produced from +the same in-memory report, never by re-reading the JSON. +""" + +import os +from typing import Any, Dict, Optional + +from ai_checker.analysis_models import AnalysisResults +from ai_checker.guidelines_reader import GuidelinesReader +from ai_checker.reports.base import ReportRenderer +from ai_checker.reports.html_renderer import HtmlRenderer +from ai_checker.reports.json_renderer import JsonRenderer +from ai_checker.reports.metadata import get_git_hash, get_timestamp +from ai_checker.reports.models import AnalysisReport, ReportMetadata +from ai_checker.reports.rst_renderer import RstRenderer + +import logging + +# Extension -> renderer class. Anything else falls back to JSON with a warning. +_RENDERERS: dict[str, type[ReportRenderer]] = { + ".html": HtmlRenderer, + ".rst": RstRenderer, + ".json": JsonRenderer, +} + +logger = logging.getLogger(__name__) + + +class ResultFormatter: + """Builds an :class:`AnalysisReport` and writes it in the requested format.""" + + def __init__( + self, + analysis_results: AnalysisResults, + model_name: Optional[str] = None, + guidelines_reader: Optional[GuidelinesReader] = None, + guidelines_output_dir: Optional[str] = None, + original_requirements: Optional[Dict[str, Dict[str, Any]]] = None, + artefact_type: str = "requirements", + ): + """ + Args: + analysis_results: AnalysisResults object containing analyses + model_name: Name of the AI model used for analysis + guidelines_reader: GuidelinesReader holding the guideline texts + guidelines_output_dir: Optional directory for guideline subpages + original_requirements: Original artefact data {id: {metadata}}; the + full ``description`` from here replaces the AI's brief one. + artefact_type: 'requirements' or 'architecture' (report metadata) + """ + self.guidelines_output_dir = guidelines_output_dir + self.report = self._build_report( + analysis_results, + model_name or "Unknown", + guidelines_reader, + original_requirements or {}, + artefact_type, + ) + + @staticmethod + def _build_report( + analysis_results: AnalysisResults, + model_name: str, + guidelines_reader: Optional[GuidelinesReader], + original_requirements: Dict[str, Dict[str, Any]], + artefact_type: str, + ) -> AnalysisReport: + # Swap each analysis's brief description for the full original when known. + analyses = [] + for analysis in analysis_results.analyses: + original = original_requirements.get(analysis.requirement_id, {}) + full_desc = original.get("description") + if full_desc: + analyses.append(analysis.model_copy(update={"description": full_desc})) + else: + analyses.append(analysis) + + guidelines = dict(guidelines_reader.guidelines) if guidelines_reader else {} + + return AnalysisReport( + metadata=ReportMetadata( + model_name=model_name, + timestamp=get_timestamp(), + git_hash=get_git_hash(), + artefact_type=artefact_type, + ), + guidelines=guidelines, + analyses=analyses, + ) + + def output(self, file_path: Optional[str] = None) -> None: + """Write the report. ``None`` prints the JSON envelope to stdout. + + The output format is chosen by ``file_path``'s extension (``.html``, + ``.rst``, else JSON). + """ + if file_path is None: + print(JsonRenderer().render(self.report)) + return + + parent = os.path.dirname(file_path) + if parent: + os.makedirs(parent, exist_ok=True) + + extension = os.path.splitext(file_path)[1].lower() + renderer = self._make_renderer(extension) + + # Every renderer accepts an output path so it can compute relative links + # to companion files; renderers that emit a self-contained document + # simply ignore it. + renderer._out_path = file_path + + with open(file_path, "w", encoding="utf-8") as f: + f.write(renderer.render(self.report)) + renderer.write_extras(self.report, file_path) + + # Under a Bazel test the absolute sandbox path is noise (and there is one + # such line per renderer); the orchestrator prints a single + # workspace-relative location instead. Only report the path for direct + # CLI use, where the absolute path is genuinely useful. + if not os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR"): + print(f"Analysis results written to {file_path}") + + def _make_renderer(self, extension: str) -> ReportRenderer: + renderer_cls = _RENDERERS.get(extension) + if renderer_cls is None: + logger.warning( + "Unknown report extension %r — writing JSON content to this " + "file regardless of its extension.", + extension, + ) + renderer_cls = JsonRenderer + # All renderers share the base constructor signature; those that don't + # emit companion guideline pages simply ignore the directory. + return renderer_cls(guidelines_output_dir=self.guidelines_output_dir) diff --git a/validation/ai_checker/src/ai_checker/reports/html_renderer.py b/validation/ai_checker/src/ai_checker/reports/html_renderer.py new file mode 100644 index 00000000..ce5c0d53 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/html_renderer.py @@ -0,0 +1,129 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""HTML renderer for analysis reports (Jinja2 template + autoescaping).""" + +import os + +from jinja2 import Environment, FileSystemLoader, select_autoescape +from markupsafe import Markup + +from ai_checker.reports.base import ReportRenderer +from ai_checker.reports.models import AnalysisReport +from ai_checker.reports.text_utils import ( + extract_severity, + markdown_to_html, + normalize_filename, + text_to_html, +) + +# Score buckets (out of 10) used for the colour-coded score badge. +_SCORE_HIGH_THRESHOLD = 8.0 +_SCORE_MEDIUM_THRESHOLD = 5.0 + +_TEMPLATE_NAME = "report.html.j2" +_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _score_class(score: float) -> str: + """Map a 0-10 score to a CSS class: ``high`` / ``medium`` / ``low``.""" + if score >= _SCORE_HIGH_THRESHOLD: + return "high" + if score >= _SCORE_MEDIUM_THRESHOLD: + return "medium" + return "low" + + +def _build_environment() -> Environment: + """Create the Jinja2 environment with autoescaping and report filters. + + ``markdown``/``text_br`` produce HTML deliberately, so they return + ``Markup`` to opt out of the autoescaper — but they escape their input + first (see ``text_utils``), so untrusted model text cannot inject HTML. + Every other interpolation is autoescaped by default. + """ + env = Environment( + loader=FileSystemLoader(_TEMPLATE_DIR), + autoescape=select_autoescape(["html", "j2"]), + trim_blocks=True, + lstrip_blocks=True, + ) + env.filters["markdown"] = lambda text: Markup(markdown_to_html(text)) + env.filters["text_br"] = lambda text: Markup(text_to_html(text)) + env.filters["severity"] = extract_severity + env.filters["score_class"] = _score_class + return env + + +class HtmlRenderer(ReportRenderer): + """Render an analysis report as a styled, self-contained HTML page.""" + + extension = ".html" + + def __init__(self, guidelines_output_dir: str | None = None): + # When set, guideline subpages are written here instead of a "guidelines" + # directory beside the report. + super().__init__(guidelines_output_dir=guidelines_output_dir) + self._template = _build_environment().get_template(_TEMPLATE_NAME) + + def render(self, report: AnalysisReport) -> str: + analyses = report.analyses + total = len(analyses) + avg_score = sum(a.score for a in analyses) / total if total else 0 + + return self._template.render( + artefact_type_title=report.metadata.artefact_type.capitalize(), + metadata=report.metadata, + total=total, + avg_score=avg_score, + guideline_links=self._guideline_links(report), + analyses=analyses, + ) + + def _guidelines_dir(self, out_path: str) -> str: + if self._guidelines_output_dir: + return self._guidelines_output_dir + return os.path.join(os.path.dirname(out_path), "guidelines") + + def _guideline_links(self, report: AnalysisReport) -> list[dict[str, str]]: + """Return ``[{name, href}]`` for each guideline page (autoescaped later).""" + if not report.guidelines: + return [] + + if self._out_path: + report_dir = os.path.dirname(self._out_path) + output_dir = self._guidelines_dir(self._out_path) + try: + relative_base = os.path.relpath(output_dir, report_dir) + except ValueError: + relative_base = output_dir + else: + relative_base = "guidelines" + + links = [] + for name in sorted(report.guidelines): + slug = normalize_filename(name) + links.append({"name": name, "href": f"{relative_base}/guideline_{slug}.md"}) + return links + + def write_extras(self, report: AnalysisReport, out_path: str) -> None: + """Write one ``guideline_.md`` page per guideline.""" + if not report.guidelines: + return + output_dir = self._guidelines_dir(out_path) + os.makedirs(output_dir, exist_ok=True) + for name, content in report.guidelines.items(): + slug = normalize_filename(name) + page_path = os.path.join(output_dir, f"guideline_{slug}.md") + with open(page_path, "w", encoding="utf-8") as f: + f.write(f"# {name}\n\n") + f.write(content) diff --git a/validation/ai_checker/src/ai_checker/reports/json_renderer.py b/validation/ai_checker/src/ai_checker/reports/json_renderer.py new file mode 100644 index 00000000..aaf907da --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/json_renderer.py @@ -0,0 +1,29 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""JSON renderer — emits the self-contained report envelope.""" + +from ai_checker.reports.base import ReportRenderer +from ai_checker.reports.models import AnalysisReport + + +class JsonRenderer(ReportRenderer): + """Serialise the full ``AnalysisReport`` envelope as indented JSON. + + The top-level ``analyses`` key is preserved (the Bazel score-threshold test + reads ``data['analyses'][].score``). + """ + + extension = ".json" + + def render(self, report: AnalysisReport) -> str: + return report.model_dump_json(indent=2) diff --git a/validation/ai_checker/src/ai_checker/reports/metadata.py b/validation/ai_checker/src/ai_checker/reports/metadata.py new file mode 100644 index 00000000..80221d2f --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/metadata.py @@ -0,0 +1,50 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Helpers for report metadata (git hash, timestamp).""" + +import os +import subprocess +from datetime import datetime + + +def get_git_hash() -> str: + """Return the current git commit hash (8 chars) or 'Unknown'. + + Prefers the Bazel workspace-status stamp variables injected via + ``--workspace_status_command``; falls back to ``git rev-parse HEAD`` in the + source tree, and finally 'Unknown' (e.g. inside a hermetic Bazel action + without git access). + """ + for env_var in ("STABLE_GIT_COMMIT", "BUILD_EMBED_LABEL", "GIT_COMMIT"): + value = os.environ.get(env_var, "").strip() + if value: + return value[:8] + + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=5, + cwd=os.path.dirname(os.path.abspath(__file__)), + ) + if result.returncode == 0: + return result.stdout.strip()[:8] + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + pass + return "Unknown" + + +def get_timestamp() -> str: + """Return the current timestamp as an ISO string (seconds precision).""" + return datetime.now().isoformat(timespec="seconds") diff --git a/validation/ai_checker/src/ai_checker/reports/models.py b/validation/ai_checker/src/ai_checker/reports/models.py new file mode 100644 index 00000000..eee49be2 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/models.py @@ -0,0 +1,54 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +Self-contained report model. + +``AnalysisReport`` is the canonical, fully self-contained representation of a +finished analysis: report-wide metadata, the guideline texts used, and the +per-requirement analyses. Everything a renderer needs lives here, so the same +object can be rendered to JSON / HTML / reST without any external state. + +Each per-requirement record is the existing ``RequirementAnalysis``; its +``description`` field holds the full original artefact description (the formatter +swaps the AI's brief description for the original when one is available). +""" + +from pydantic import BaseModel, Field + +from ai_checker.analysis_models import RequirementAnalysis + + +class ReportMetadata(BaseModel): + """Report-wide metadata.""" + + model_name: str = Field(description="AI model used for the analysis") + timestamp: str = Field(description="ISO timestamp when the report was produced") + git_hash: str = Field(description="Git commit hash of the analyzed sources") + artefact_type: str = Field( + default="requirements", + description="Artefact type analyzed ('requirements' or 'architecture')", + ) + + +class AnalysisReport(BaseModel): + """Everything needed to render an analysis report.""" + + metadata: ReportMetadata + guidelines: dict[str, str] = Field( + default_factory=dict, + description="Guideline name -> markdown content (for subpages and links)", + ) + analyses: list[RequirementAnalysis] = Field( + default_factory=list, + description="Per-requirement analyses (description = full original text)", + ) diff --git a/validation/ai_checker/src/ai_checker/reports/report.html.j2 b/validation/ai_checker/src/ai_checker/reports/report.html.j2 new file mode 100644 index 00000000..89b184c2 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/report.html.j2 @@ -0,0 +1,196 @@ + + + + + + + Analysis Report + + + +
+

{{ artefact_type_title }} Analysis Report

+

Comprehensive analysis of artefacts against engineering guidelines

+
+

Hash: {{ metadata.git_hash }}

+

Timestamp: {{ metadata.timestamp }}

+
+
+ +
+
+

Total Artefacts

+
{{ total }}
+
+
+

Average Score

+
{{ "%.1f"|format(avg_score) }}/10
+
+
+

AI Model Used

+
{{ metadata.model_name }}
+
+
+ +
+
+

Guidelines Used

+
+
    +{% if guideline_links %} +{% for link in guideline_links %} +
  • 📋 {{ link.name }}
    • +{% endfor %} +{% else %} +
  • No guidelines specified
    • +{% endif %} +
+
+
+
+ +
+{% for analysis in analyses %} +
+
+
{{ analysis.requirement_id }}
+
{{ "%.1f"|format(analysis.score) }}/10
+
+ +
+ Description: {{ analysis.description|text_br }} +
+{% if analysis.findings %} +
+

Findings

+
    +{% for finding in analysis.findings %} +
  • {{ finding|markdown }}
    • +{% endfor %} +
+
+{% endif %} +{% if analysis.suggestions %} +
+

Suggestions

+
    +{% for suggestion in analysis.suggestions %} +
  • {{ suggestion|markdown }}
    • +{% endfor %} +
+
+{% endif %} +
+{% endfor %} +
+ + diff --git a/validation/ai_checker/src/ai_checker/reports/rst_renderer.py b/validation/ai_checker/src/ai_checker/reports/rst_renderer.py new file mode 100644 index 00000000..c6d3f4c3 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/rst_renderer.py @@ -0,0 +1,122 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Standalone reStructuredText renderer for analysis reports.""" + +import os + +from ai_checker.reports.base import ReportRenderer +from ai_checker.reports.models import AnalysisReport +from ai_checker.reports.text_utils import normalize_filename, strip_markup + + +def _heading(text: str, char: str) -> str: + """reST heading: text on one line, underline of ``char`` of equal length.""" + return f"{text}\n{char * len(text)}\n" + + +def _bullets(items: list[str]) -> str: + """Render a reST bullet list.""" + lines = [] + for item in items: + text = strip_markup(item).replace("\n", " ") + lines.append(f"- {text}") + return "\n".join(lines) + "\n" + + +class RstRenderer(ReportRenderer): + """Render an analysis report as standalone reStructuredText (no docutils).""" + + extension = ".rst" + + def __init__(self, guidelines_output_dir: str | None = None): + super().__init__(guidelines_output_dir=guidelines_output_dir) + + def render(self, report: AnalysisReport) -> str: + analyses = report.analyses + total = len(analyses) + avg_score = sum(a.score for a in analyses) / total if total else 0 + meta = report.metadata + + parts: list[str] = [] + title = f"{meta.artefact_type.capitalize()} Analysis Report" + parts.append(_heading(title, "=")) + parts.append("") + parts.append(f":Total artefacts: {total}") + parts.append(f":Average score: {avg_score:.1f}/10") + parts.append(f":AI model: {meta.model_name}") + parts.append(f":Hash: {meta.git_hash}") + parts.append(f":Timestamp: {meta.timestamp}") + parts.append("") + + # Guidelines + parts.append(_heading("Guidelines used", "-")) + if report.guidelines: + base = self._relative_base() + for name in sorted(report.guidelines): + slug = normalize_filename(name) + parts.append(f"- `{name} <{base}/guideline_{slug}.rst>`_") + else: + parts.append("- No guidelines specified") + parts.append("") + + # Per-artefact sections + for analysis in analyses: + parts.append( + _heading(f"{analysis.requirement_id} ({analysis.score:.1f}/10)", "-") + ) + parts.append("") + desc = strip_markup(analysis.description).replace("\n", " ") + parts.append(f"**Description:** {desc}") + parts.append("") + if analysis.findings: + parts.append("Findings:") + parts.append("") + parts.append(_bullets(analysis.findings)) + if analysis.suggestions: + parts.append("Suggestions:") + parts.append("") + parts.append(_bullets(analysis.suggestions)) + + return "\n".join(parts).rstrip() + "\n" + + def _guidelines_dir(self, out_path: str) -> str: + if self._guidelines_output_dir: + return self._guidelines_output_dir + return os.path.join(os.path.dirname(out_path), "guidelines") + + def _relative_base(self) -> str: + if self._out_path: + report_dir = os.path.dirname(self._out_path) + output_dir = self._guidelines_dir(self._out_path) + try: + return os.path.relpath(output_dir, report_dir) + except ValueError: + return output_dir + return "guidelines" + + def write_extras(self, report: AnalysisReport, out_path: str) -> None: + """Write one ``guideline_.rst`` page per guideline.""" + if not report.guidelines: + return + output_dir = self._guidelines_dir(out_path) + os.makedirs(output_dir, exist_ok=True) + for name, content in report.guidelines.items(): + slug = normalize_filename(name) + page_path = os.path.join(output_dir, f"guideline_{slug}.rst") + with open(page_path, "w", encoding="utf-8") as f: + f.write(_heading(name, "=")) + f.write("\n::\n\n") + # Embed source guideline verbatim as a literal block (it is + # markdown, not reST, so quote it to avoid parse errors). + for line in content.splitlines(): + f.write(f" {line}\n") diff --git a/validation/ai_checker/src/ai_checker/reports/text_utils.py b/validation/ai_checker/src/ai_checker/reports/text_utils.py new file mode 100644 index 00000000..37e19e22 --- /dev/null +++ b/validation/ai_checker/src/ai_checker/reports/text_utils.py @@ -0,0 +1,102 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Shared text helpers for report renderers (slugs, severity, markdown→HTML).""" + +import html +import re + + +def normalize_filename(text: str) -> str: + """Convert text to a filesystem-safe filename slug. + + Lowercase, spaces/underscores to hyphens, unsafe characters removed, + collapsed and trimmed hyphens. + """ + slug = text.lower() + slug = re.sub(r"[\s_]+", "-", slug) + slug = re.sub(r"[^a-z0-9-]", "", slug) + slug = slug.strip("-") + slug = re.sub(r"-+", "-", slug) + return slug + + +def extract_severity(finding: str) -> str: + """Extract a severity CSS class from finding text. + + The severity word (``Major`` / ``Minor``) may be wrapped in markdown + (``**...**``) or HTML (```` / ````, escaped or not) and may be + followed by a colon, an en-dash summary (``Minor – …:``) or other text. + Strip any leading emphasis marker, then read the first word. + + Returns 'major', 'minor', or '' when no severity is found. + """ + # Drop a leading emphasis marker: ** , , or their escaped forms. + stripped = re.sub( + r"^\s*(?:\*\*|<(?:b|strong)>|<(?:b|strong)>)\s*", + "", + finding, + flags=re.IGNORECASE, + ) + match = re.match(r"(Major|Minor)\b", stripped, re.IGNORECASE) + if match: + return match.group(1).lower() + return "" + + +# Inline HTML tags the model is allowed to emit in findings/suggestions. After +# escaping the whole string (so arbitrary/unsafe HTML cannot pass through), +# only these bare tags are re-enabled — no attributes are un-escaped, so this +# stays XSS-safe. +_ALLOWED_INLINE_TAGS = ("b", "strong", "i", "em", "u", "code") +_ESCAPED_TAG_RE = re.compile( + r"<(/?(?:" + "|".join(_ALLOWED_INLINE_TAGS) + r"))>", + re.IGNORECASE, +) + + +def markdown_to_html(text: str) -> str: + """Convert markdown emphasis to HTML, escaping everything else. + + The model sometimes emits a small set of inline HTML tags (e.g. ````) + directly. The whole string is escaped first (preventing HTML injection), + then markdown ``**bold**`` / ``*italic*`` are translated and the allowlisted + inline tags are re-enabled so they render instead of showing as literal + ``<b>`` text. + """ + text = html.escape(text) + text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) + text = re.sub(r"\*([^*]+)\*", r"\1", text) + text = _ESCAPED_TAG_RE.sub(r"<\1>", text) + text = text.replace("\n", "
\n") + return text + + +def text_to_html(text: str) -> str: + """Escape plain text and convert line breaks to ``
`` tags.""" + escaped = html.escape(text) + return escaped.replace("\n", "
\n") + + +_RST_SPECIAL = re.compile(r"^([=\-`:.'\"~^_*+#])") + + +def strip_markup(text: str) -> str: + """Reduce markdown/HTML formatting to plain text for reST output. + + reST is plain-text; we drop bold/italic markers and HTML tags rather than + translate them, keeping the content readable and valid. + """ + text = re.sub(r"<[^>]+>", "", text) # drop HTML tags + text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) # **bold** + text = re.sub(r"\*([^*]+)\*", r"\1", text) # *italic* + return text.strip() diff --git a/validation/ai_checker/src/ai_checker/result_formatter.py b/validation/ai_checker/src/ai_checker/result_formatter.py deleted file mode 100644 index afc41a85..00000000 --- a/validation/ai_checker/src/ai_checker/result_formatter.py +++ /dev/null @@ -1,607 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -""" -Result formatter for TRLC AI Checker analysis results. - -This module provides formatting and output functionality for analysis results -in various formats (stdout, JSON, HTML). -""" - -import html -import os -import re -import subprocess -from datetime import datetime -from typing import Any, Dict, Optional - -from ai_checker.analysis_models import AnalysisResults -from ai_checker.guidelines_reader import GuidelinesReader - - -class ResultFormatter: - """ - Handles formatting and output of analysis results in multiple formats. - """ - - @staticmethod - def _get_git_hash() -> str: - """Get the current git commit hash. - - First checks the BUILD_EMBED_LABEL / STABLE_GIT_COMMIT stamp variables - injected by Bazel --workspace_status_command. Falls back to running - ``git rev-parse HEAD`` in the source tree, and finally returns - 'Unknown' if neither is available (e.g. inside a fully hermetised - Bazel action without network/git access). - - Returns: - Git commit hash (8 chars) or 'Unknown' - """ - # Prefer Bazel workspace-status stamp variables (set by --workspace_status_command) - for env_var in ("STABLE_GIT_COMMIT", "BUILD_EMBED_LABEL", "GIT_COMMIT"): - value = os.environ.get(env_var, "").strip() - if value: - return value[:8] - - try: - # Fall back to running git directly (works for local CLI invocations) - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - capture_output=True, - text=True, - timeout=5, - cwd=os.path.dirname(os.path.abspath(__file__)), - ) - if result.returncode == 0: - return result.stdout.strip()[:8] - except (FileNotFoundError, subprocess.TimeoutExpired, OSError): - pass - return "Unknown" - - @staticmethod - def _get_timestamp() -> str: - """Get the current timestamp. - - Returns: - ISO format timestamp string - """ - return datetime.now().isoformat(timespec="seconds") - - @staticmethod - def _normalize_filename(text: str) -> str: - """Convert text to filesystem-safe filename slug. - - Args: - text: Text to normalize - - Returns: - Normalized string (lowercase, spaces to hyphens, unsafe chars removed) - """ - # Convert to lowercase - slug = text.lower() - # Replace spaces and underscores with hyphens - slug = re.sub(r"[\s_]+", "-", slug) - # Remove unsafe characters (keep alphanumeric and hyphens) - slug = re.sub(r"[^a-z0-9-]", "", slug) - # Remove leading/trailing hyphens - slug = slug.strip("-") - # Collapse multiple hyphens - slug = re.sub(r"-+", "-", slug) - return slug - - @staticmethod - def _extract_severity(finding: str) -> str: - """Extract severity level from finding text. - - Args: - finding: Finding text that may start with Major:, Minor:, **Major**:, Major:, or Major: - - Returns: - CSS class name: 'major', 'minor', or empty string if no severity found - """ - # Check for plain text format: Major: or Minor: - match = re.match(r"^(Major|Minor):\s", finding, re.IGNORECASE) - if match: - return match.group(1).lower() - # Check for markdown format: **Major**: or **Minor**: - match = re.match(r"^\*\*(Major|Minor)\*\*:", finding, re.IGNORECASE) - if match: - return match.group(1).lower() - # Check for HTML format: Major: or Minor: or Major: - match = re.match( - r"^<(?:b|strong)>(Major|Minor):", finding, re.IGNORECASE - ) - if match: - return match.group(1).lower() - # Check for escaped HTML: <b>Major:</b> or <strong>Minor:</strong> - match = re.match( - r"^<(?:b|strong)>(Major|Minor):</(?:b|strong)>", - finding, - re.IGNORECASE, - ) - if match: - return match.group(1).lower() - return "" - - @staticmethod - def _markdown_to_html(text: str) -> str: - """Convert markdown formatting to HTML while preserving existing HTML tags. - - Args: - text: Text with markdown and/or HTML formatting - - Returns: - HTML-formatted string with markdown converted and HTML preserved - """ - # First escape HTML to protect it, but mark HTML tags specially - # Replace HTML tags with placeholders - html_tags = [] - - def save_html_tag(match): - html_tags.append(match.group(0)) - return f"__HTML_TAG_{len(html_tags) - 1}__" - - # Save HTML tags - text = re.sub(r"<[^>]+>", save_html_tag, text) - - # Now escape any remaining HTML special characters - text = html.escape(text) - - # Convert markdown formatting - # Bold: **text** to text - text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) - # Italic: *text* to text - text = re.sub(r"\*([^*]+)\*", r"\1", text) - - # Restore HTML tags - for i, tag in enumerate(html_tags): - text = text.replace(f"__HTML_TAG_{i}__", tag) - - # Convert line breaks to
tags - text = text.replace("\n", "
\n") - - return text - - @staticmethod - def _text_to_html(text: str) -> str: - """Convert plain text to HTML with proper line breaks and escaping. - - Args: - text: Plain text string - - Returns: - HTML-formatted string with line breaks converted to
tags - """ - # Escape HTML special characters - escaped = html.escape(text) - # Convert line breaks to
tags - return escaped.replace("\n", "
\n") - - def __init__( - self, - analysis_results: AnalysisResults, - model_name: Optional[str] = None, - guidelines_reader: Optional[GuidelinesReader] = None, - guidelines_output_dir: Optional[str] = None, - original_requirements: Optional[Dict[str, Dict[str, Any]]] = None, - ): - """ - Initialize the formatter with analysis results. - - Args: - analysis_results: AnalysisResults object containing analyses - model_name: Name of the AI model used for analysis - guidelines_reader: GuidelinesReader object containing all guidelines - guidelines_output_dir: Optional directory path for writing guideline files - original_requirements: Original requirement data as dict {id: {metadata}} - """ - self.results = analysis_results - self.model_name = model_name or "Unknown" - self.guidelines_reader = guidelines_reader - self.guidelines_output_dir = guidelines_output_dir - self.git_hash = self._get_git_hash() - self.timestamp = self._get_timestamp() - - # Create lookup map for original requirement descriptions - self.original_descriptions = {} - if original_requirements: - for req_id, req_data in original_requirements.items(): - self.original_descriptions[req_id] = req_data.get("description", "") - - def output(self, file_path: Optional[str] = None) -> None: - """ - Output results based on file path extension or to stdout. - - Args: - file_path: Optional path to output file. If None, prints JSON to stdout. - Extension determines format: .html for HTML, otherwise JSON. - """ - if file_path is None: - self._print_to_stdout() - else: - # Create parent directories if they don't exist - parent = os.path.dirname(file_path) - if parent: - os.makedirs(parent, exist_ok=True) - - extension = os.path.splitext(file_path)[1].lower() - - if extension == ".html": - self._write_html(file_path) - else: - self._write_json(file_path) - - print(f"Analysis results written to {file_path}") - - def _print_to_stdout(self) -> None: - """Print results as JSON to stdout.""" - output = self.results.model_dump_json(indent=2) - print(output) - - def _write_json(self, path: str) -> None: - """ - Write results as JSON file. - - Args: - path: File path for output JSON file - """ - with open(path, "w", encoding="utf-8") as f: - f.write(self.results.model_dump_json(indent=2)) - - def _write_html(self, path: str) -> None: - """ - Write results as HTML report with guideline subpages. - - Args: - path: File path for output HTML file - """ - - # Generate main report - html_content = self._generate_html_report(path) - with open(path, "w", encoding="utf-8") as f: - f.write(html_content) - - # Generate guideline subpages - if self.guidelines_reader: - self._generate_guideline_pages(path) - - def _generate_html_report(self, main_report_path: Optional[str] = None) -> str: - """ - Generate HTML report from analysis results. - - Args: - main_report_path: Optional path to main report file for computing relative links - - Returns: - HTML string containing formatted report - """ - # Calculate summary statistics - total_requirements = len(self.results.analyses) - avg_score = ( - sum(a.score for a in self.results.analyses) / total_requirements - if total_requirements > 0 - else 0 - ) - - # Escape all untrusted values before interpolating into HTML - safe_git_hash = html.escape(self.git_hash) - safe_timestamp = html.escape(self.timestamp) - safe_model_name = html.escape(self.model_name) - - doc = f""" - - - - - Requirements Analysis Report - - - -
-

Requirements Analysis Report

-

Comprehensive analysis of requirements against engineering guidelines

-
-

Hash: {safe_git_hash}

-

Timestamp: {safe_timestamp}

-
-
- -
-
-

Total Requirements

-
{total_requirements}
-
-
-

Average Score

-
{avg_score:.1f}/10
-
-
-

AI Model Used

-
{safe_model_name}
-
-
- -
-
-

Guidelines Used

-
-
    -{self._generate_guidelines_links(main_report_path)} -
-
-
-
- -
-""" - - # Add individual requirement sections - for analysis in self.results.analyses: - score_class = ( - "high" - if analysis.score >= 8 - else "medium" - if analysis.score >= 5 - else "low" - ) - - # Use original full description if available, otherwise use - # AI's description - description = self.original_descriptions.get( - analysis.requirement_id, analysis.description - ) - # Escape requirement_id: it comes from user-supplied TRLC files - safe_req_id = html.escape(analysis.requirement_id) - - doc += f""" -
-
-
{safe_req_id}
-
{analysis.score:.1f}/10
-
- -
- Description: {self._text_to_html(description)} -
-""" - - if analysis.findings: - doc += """ -
-

Findings

-
    -""" - for finding in analysis.findings: - severity_class = self._extract_severity(finding) - formatted_finding = self._markdown_to_html(finding) - doc += f'
  • {formatted_finding}
    • \n' - doc += """
-
-""" - - if analysis.suggestions: - doc += """ -
-

Suggestions

-
    -""" - for suggestion in analysis.suggestions: - doc += f"
  • {self._markdown_to_html(suggestion)}
    • \n" - doc += """
-
-""" - - doc += """
-""" - - doc += """
- - -""" - return doc - - def _generate_guidelines_links(self, main_report_path: Optional[str] = None) -> str: - """Generate HTML for guidelines list with links to subpages. - - Args: - main_report_path: Path to main report (used to compute relative paths) - - Returns: - HTML string with list items for guidelines - """ - if not self.guidelines_reader or not self.guidelines_reader.guidelines: - return ( - '
  • No guidelines specified
    • ' - ) - - # Compute output directory for guidelines (same logic as _generate_guideline_pages) - if main_report_path: - report_dir = os.path.dirname(main_report_path) - if self.guidelines_output_dir: - output_dir = self.guidelines_output_dir - else: - output_dir = os.path.join(report_dir, "guidelines") - - # Compute relative path from report directory to guidelines directory - try: - relative_base_str = os.path.relpath(output_dir, report_dir) - except ValueError: - # If not relative, use absolute path - relative_base_str = output_dir - else: - # Fallback to hardcoded path if no main_report_path provided - relative_base_str = "guidelines" - - links = [] - for guideline_name in sorted(self.guidelines_reader.guidelines.keys()): - # Normalize guideline name for filename - slug = self._normalize_filename(guideline_name) - links.append( - f'
  • ' - f'' - f"📋 {guideline_name}
    • " - ) - return "\n".join(links) - - def _generate_guideline_pages(self, main_report_path: str) -> None: - """Generate markdown files for each guideline. - - Args: - main_report_path: Path to main report (used to determine output directory) - """ - if not self.guidelines_reader: - return - - # Use guidelines subdirectory in the same parent directory as main report - if self.guidelines_output_dir: - output_dir = self.guidelines_output_dir - else: - output_dir = os.path.join(os.path.dirname(main_report_path), "guidelines") - - os.makedirs(output_dir, exist_ok=True) - - for ( - guideline_name, - guideline_content, - ) in self.guidelines_reader.guidelines.items(): - # Normalize guideline name for filename - slug = self._normalize_filename(guideline_name) - page_path = os.path.join(output_dir, f"guideline_{slug}.md") - with open(page_path, "w", encoding="utf-8") as f: - f.write(f"# {guideline_name}\n\n") - f.write(guideline_content) diff --git a/validation/ai_checker/src/copilot_adapter/_message_converter.py b/validation/ai_checker/src/copilot_adapter/_message_converter.py deleted file mode 100644 index 8a1d4d3b..00000000 --- a/validation/ai_checker/src/copilot_adapter/_message_converter.py +++ /dev/null @@ -1,68 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -"""Conversion between LangChain message types and Copilot SDK prompt format.""" - -from __future__ import annotations - -import json -from typing import Optional - -from langchain_core.messages import ( - AIMessage, - BaseMessage, - HumanMessage, - SystemMessage, - ToolMessage, -) - - -def messages_to_prompt(messages: list[BaseMessage]) -> str: - """Convert a list of LangChain messages into a single prompt string. - - The Copilot SDK accepts a plain text prompt rather than a structured - message array. We serialise the conversation into a tagged format so - the model can distinguish roles. - """ - parts: list[str] = [] - for msg in messages: - content = ( - msg.content if isinstance(msg.content, str) else json.dumps(msg.content) - ) - - if isinstance(msg, SystemMessage): - parts.append(f"[system]\n{content}") - elif isinstance(msg, HumanMessage): - parts.append(f"[user]\n{content}") - elif isinstance(msg, AIMessage): - text_parts = [f"[assistant]\n{content}"] if content else ["[assistant]"] - if msg.tool_calls: - for tc in msg.tool_calls: - text_parts.append( - f"[tool_call id={tc['id']} name={tc['name']}]\n" - f"{json.dumps(tc['args'])}" - ) - parts.append("\n".join(text_parts)) - elif isinstance(msg, ToolMessage): - parts.append(f"[tool_result id={msg.tool_call_id}]\n{content}") - else: - parts.append(f"[{msg.type}]\n{content}") - - return "\n\n".join(parts) - - -def extract_system_message(messages: list[BaseMessage]) -> Optional[str]: - """Return the content of the first message if it is a SystemMessage.""" - if messages and isinstance(messages[0], SystemMessage): - content = messages[0].content - return content if isinstance(content, str) else json.dumps(content) - return None diff --git a/validation/ai_checker/src/copilot_adapter/_tool_converter.py b/validation/ai_checker/src/copilot_adapter/_tool_converter.py deleted file mode 100644 index 026b1a4a..00000000 --- a/validation/ai_checker/src/copilot_adapter/_tool_converter.py +++ /dev/null @@ -1,97 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -"""Conversion between LangChain tool specs and Copilot SDK Tool objects.""" - -from __future__ import annotations - -import json -from collections.abc import Sequence -from typing import Any, Callable - -from copilot.tools import Tool as CopilotTool, ToolInvocation, ToolResult -from langchain_core.tools import BaseTool -from langchain_core.utils.function_calling import convert_to_openai_tool - - -def convert_tools_to_openai_format( - tools: Sequence[dict[str, Any] | type | Callable | BaseTool], -) -> list[dict[str, Any]]: - """Convert LangChain tool specs to OpenAI-format tool definitions.""" - result = [] - for tool in tools: - if isinstance(tool, dict): - result.append(tool) - else: - result.append(convert_to_openai_tool(tool)) - return result - - -def build_copilot_tools( - openai_tools: list[dict[str, Any]], -) -> list[CopilotTool]: - """Convert OpenAI-format tool dicts into Copilot SDK Tool objects. - - The handler is a no-op because we never let the Copilot agent - autonomously execute tools — we only need the definitions so the - model can emit tool_calls in its response. - """ - copilot_tools = [] - for t in openai_tools: - fn = t.get("function", t) - name = fn["name"] - description = fn.get("description", "") - parameters = fn.get("parameters") - - def _make_noop_handler(tool_name: str): - async def _noop_handler(invocation: ToolInvocation) -> ToolResult: - return ToolResult( - text_result_for_llm="Tool execution is managed by LangChain.", - result_type="success", - ) - - return _noop_handler - - copilot_tools.append( - CopilotTool( - name=name, - description=description, - handler=_make_noop_handler(name), - parameters=parameters, - ) - ) - return copilot_tools - - -def deep_decode_json_strings(obj: Any) -> Any: - """Recursively decode values that are JSON-encoded strings. - - Some LLMs (e.g. Claude via the Copilot SDK) double-encode nested - lists or objects as JSON strings inside the outer tool-call arguments - dict. This function walks the structure and replaces any string value - that successfully parses as a JSON array or object with the decoded - Python value, leaving plain strings untouched. - """ - if isinstance(obj, dict): - return {k: deep_decode_json_strings(v) for k, v in obj.items()} - if isinstance(obj, list): - return [deep_decode_json_strings(v) for v in obj] - if isinstance(obj, str): - stripped = obj.strip() - if stripped and stripped[0] in ("{", "["): - try: - decoded = json.loads(stripped) - if isinstance(decoded, (dict, list)): - return deep_decode_json_strings(decoded) - except (json.JSONDecodeError, ValueError): - pass - return obj diff --git a/validation/ai_checker/src/copilot_adapter/architecture.md b/validation/ai_checker/src/copilot_adapter/architecture.md deleted file mode 100644 index ebd3bea3..00000000 --- a/validation/ai_checker/src/copilot_adapter/architecture.md +++ /dev/null @@ -1,206 +0,0 @@ - - -# copilot_adapter — Architecture - -## Overview - -`copilot_adapter` is a [LangChain](https://python.langchain.com/) integration layer that -bridges the **GitHub Copilot SDK** (`github-copilot-sdk`) to the LangChain ecosystem. -It exposes a single public class, `ChatCopilot`, which is a drop-in replacement for any -other LangChain `BaseChatModel` (e.g. `ChatOpenAI`). - -The adapter translates between two different worlds: - -| LangChain side | Copilot SDK side | -|---|---| -| `list[BaseMessage]` (typed message objects) | A single plain-text prompt string with role tags | -| `SystemMessage` | `SessionConfig.system_message` (injected once per session) | -| `BaseTool` / OpenAI tool dict | `copilot.tools.Tool` with async handler | -| Pydantic `BaseModel` schema | JSON schema embedded in the system prompt | -| `AIMessage` with `tool_calls` | `ExternalToolRequestedData` broadcast events | - ---- - -## Component Diagram - - - ---- - -## Module Responsibilities - -### `copilot_langchain.py` — `ChatCopilot` - -The central public class. Inherits from LangChain's `BaseChatModel` so it can be used -anywhere a standard LangChain model is expected. - -| Method | Role | -|---|---| -| `with_structured_output(schema)` | Returns a composed `Runnable` chain for structured JSON output (see below) | -| `bind_tools(tools)` | Returns a new `ChatCopilot` instance with the given tools pre-registered | -| `_agenerate(messages)` | **Async core** — creates a Copilot session, sends the prompt, collects the response | -| `_generate(messages)` | Sync bridge: runs `_agenerate` in a thread-pool executor if an event loop is already running | - -### `_client_manager.py` — `CopilotClientManager` - -Owns the lifecycle of the single `CopilotClient` / CLI subprocess. The same subprocess -is reused across calls (cached in `_client`). - -Pre-flight sequence executed once before the first request: -1. Resolve the `copilot_cli` binary path (Bazel `copy_executables` workaround) -2. Verify the binary exists and is executable -3. Hard-fail if no auth source is found (`COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`, or `~/.copilot/config.json`) -4. Warn (non-fatal) about missing `$HOME` or `HTTPS_PROXY` -5. Spawn the subprocess and authenticate via `get_auth_status()` - -### `_message_converter.py` - -Converts the LangChain message list into the formats the Copilot SDK accepts. - -- **`extract_system_message(messages)`** — Pulls out the first `SystemMessage` and - returns its string content. This is passed as `SessionConfig.system_message` so the - Copilot CLI handles it as a true system prompt (not just prepended text). - -- **`messages_to_prompt(messages)`** — Serialises all remaining messages into a single - tagged plain-text string: - ``` - [user] - What is 2 + 2? - - [assistant] - 4 - [tool_call id=abc name=add] - {"a": 2, "b": 2} - - [tool_result id=abc] - 4 - ``` - -### `_tool_converter.py` - -Converts tool definitions between three representations: - -``` -LangChain BaseTool / Callable / type - │ - ▼ convert_to_openai_tool() -OpenAI function dict {"type": "function", "function": {"name": ..., "parameters": ...}} - │ - ▼ build_copilot_tools() -copilot.tools.Tool (with async no-op handler) -``` - -The handler is always a no-op because when using `with_structured_output` tool execution -is bypassed entirely; when using `bind_tools` directly, tool execution is managed by the -LangChain agent loop, not by the Copilot CLI. - -`deep_decode_json_strings` recursively unwraps values that the model has -double-encoded as JSON strings inside the outer tool-call arguments dict. - -### `_preflight.py` - -Stateless helper functions called by `CopilotClientManager` before startup: - -- `resolve_copilot_cli_path()` — walks up from `copilot.__file__` to find the - `copilot_cli` binary that Bazel's `copy_executables` placed next to the package. -- `check_cli_binary(path)` — checks existence and executable bit. -- `check_auth_sources()` — scans env vars and `~/.copilot/config.json`; hard-fails if - none are present. -- `check_environment()` — warns about missing `HOME`, `HTTPS_PROXY`, or proxy vars. -- `describe_auth_sources()` — formats a human-readable description for error messages. - -### `_errors.py` - -- `CopilotSetupError` — a `RuntimeError` subclass raised for any configuration or - startup failure. Carries an actionable message for the user. -- `AUTH_ENV_VARS` — ordered list of accepted auth environment variables. - ---- - -## Data Flow: `with_structured_output` - -The primary usage path (used by `ai_checker_core`). `with_structured_output(schema)` -returns a composed chain: `_inject | ChatCopilot | _parse`. - -1. **`_inject`** — appends the Pydantic schema (serialised to JSON) to the - `SystemMessage`, instructing the model to respond with only a matching JSON object. -2. **`ChatCopilot._agenerate`** — extracts the system message into - `SessionConfig.system_message`, serialises remaining messages into a role-tagged - plain-text prompt, creates a Copilot session, and calls `send_and_wait`. Returns an - `AIMessage` whose `.content` is the model's raw text reply. -3. **`_parse`** — strips any markdown fences, extracts the outermost `{…}` substring, - parses it with `json.loads`, and validates it with `schema.model_validate`. On any - failure it raises a `ValueError` containing the full raw LLM output and the specific - error (JSON byte position or Pydantic field detail). - -> **Why JSON-in-prompt instead of tool calling?** -> The Copilot CLI uses a Claude model with reasoning enabled. That model ignores -> `tool_choice="any"` and always responds in plain text. Embedding the JSON schema -> directly in the system prompt is reliably followed. - ---- - -## Data Flow: `bind_tools` (direct tool use) - -Used when the LangChain agent loop — not the adapter — executes tools. - -1. `bind_tools(tools)` converts each tool to OpenAI format, then to a `copilot.tools.Tool` - with a no-op handler, and registers them in `SessionConfig`. -2. Tool calls arrive as `ExternalToolRequestedData` broadcast events (SDK protocol v3) - or in `AssistantMessageData.tool_requests` (legacy fallback). -3. Both sources are merged, deduplicated by `tool_call_id`, and returned as - `AIMessage.tool_calls` for the LangChain agent loop to dispatch. - ---- - -## Authentication - -The Copilot CLI requires a valid GitHub OAuth token to contact the Copilot API. -`_preflight.py` checks the following sources in priority order during pre-flight: - -| Priority | Source | Notes | -|---|---|---| -| 1 | `COPILOT_GITHUB_TOKEN` env var | Recommended for explicit Copilot usage | -| 2 | `GH_TOKEN` env var | GitHub CLI compatible | -| 3 | `GITHUB_TOKEN` env var | GitHub Actions compatible | -| 4 | `~/.copilot/config.json` | Written by `gh copilot` interactive login | - -If **none** of these sources is present, `CopilotClientManager` raises a -`CopilotSetupError` before spawning the subprocess (hard fail — no point starting -the CLI without credentials). - -If at least one source exists, the CLI is started and `get_auth_status()` is called to -confirm the token is accepted by the GitHub API. A failed status also raises -`CopilotSetupError` with an actionable message. - -### Bazel / headless environments - -In the CI/Bazel setup the `--config=copilot` bazelrc flag forwards `HOME` and the -proxy environment variables (`HTTPS_PROXY`, `HTTP_PROXY`, `NO_PROXY`) to the action -sandbox via `--action_env`. Without `HOME` the CLI cannot locate `~/.copilot/config.json`, -so a token env var must be set instead. `_preflight.py` emits a warning (non-fatal) -when `HOME` is unset. - ---- - -## Error Handling Summary - -| Failure point | Exception type | What is logged | -|---|---|---| -| CLI binary missing / not executable | `CopilotSetupError` | Path checked, alternatives suggested | -| No auth source found | `CopilotSetupError` | Lists all env vars and config file path | -| Copilot SDK startup error | `CopilotSetupError` | Wraps original exception + auth description | -| Model returns no JSON object | `ValueError` | Full LLM output | -| Model returns malformed JSON | `ValueError` | `json.JSONDecodeError` position + full LLM output | -| Model returns wrong JSON structure | `ValueError` | Pydantic field-level `ValidationError` + full LLM output | diff --git a/validation/ai_checker/src/copilot_adapter/component_diagram.puml b/validation/ai_checker/src/copilot_adapter/component_diagram.puml deleted file mode 100644 index 58c5d70c..00000000 --- a/validation/ai_checker/src/copilot_adapter/component_diagram.puml +++ /dev/null @@ -1,61 +0,0 @@ -' ******************************************************************************* -' Copyright (c) 2026 Contributors to the Eclipse Foundation -' -' See the NOTICE file(s) distributed with this work for additional -' information regarding copyright ownership. -' -' This program and the accompanying materials are made available under the -' terms of the Apache License Version 2.0 which is available at -' https://www.apache.org/licenses/LICENSE-2.0 -' -' SPDX-License-Identifier: Apache-2.0 -' ******************************************************************************* - -@startuml copilot_adapter -!theme plain -skinparam componentStyle rectangle -skinparam defaultFontName Monospaced -skinparam ArrowFontSize 11 - -package "langchain_core" <> { - component BaseChatModel - component "Messages / Tools" as LCTypes -} - -package "github-copilot-sdk" <> { - component CopilotClient - component "Session API" as SessionAPI - component "Tool types" as CopilotTools -} - -package "pydantic" <> { - component BaseModel -} - -package "copilot_adapter" { - component ChatCopilot - component CopilotClientManager as ClientMgr - component _message_converter as MsgConv - component _tool_converter as ToolConv - component _preflight as Preflight - component _errors as Errors -} - -actor "Caller" as Caller - -Caller --> ChatCopilot : ainvoke(messages) -ChatCopilot -up-> BaseChatModel -ChatCopilot --> ClientMgr : ensure_client() -ChatCopilot --> MsgConv : convert messages -ChatCopilot --> ToolConv : convert tools -ChatCopilot --> SessionAPI : create_session / send_and_wait -ChatCopilot --> BaseModel : inject schema / validate - -ClientMgr --> CopilotClient -ClientMgr --> Preflight : pre-flight checks -ClientMgr --> Errors : CopilotSetupError - -ToolConv --> CopilotTools -ToolConv --> LCTypes -MsgConv --> LCTypes -@enduml diff --git a/validation/ai_checker/src/copilot_adapter/copilot_langchain.py b/validation/ai_checker/src/copilot_adapter/copilot_langchain.py deleted file mode 100644 index 3d419329..00000000 --- a/validation/ai_checker/src/copilot_adapter/copilot_langchain.py +++ /dev/null @@ -1,389 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -""" -LangChain BaseChatModel wrapper for the GitHub Copilot SDK. - -Provides a fully LangChain-compatible chat model that supports: -- Standard message types (system, human, AI, tool) -- Tool calling via bind_tools() -- Structured output via with_structured_output() -- Async generation (native) -- Sync generation (via asyncio bridge) -""" - -from __future__ import annotations - -import asyncio -import concurrent.futures -import json -import logging -from collections.abc import Sequence -from typing import Any, Callable, Optional - -from copilot.generated.session_events import ExternalToolRequestedData, SessionEventType -from copilot.session import PermissionHandler, SessionConfig - -from langchain_core.callbacks import ( - AsyncCallbackManagerForLLMRun, - CallbackManagerForLLMRun, -) -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import AIMessage, BaseMessage, SystemMessage -from langchain_core.outputs import ChatGeneration, ChatResult -from langchain_core.tools import BaseTool -from pydantic import Field, PrivateAttr - -from ._client_manager import CopilotClientManager -from ._errors import CopilotSetupError -from ._message_converter import extract_system_message, messages_to_prompt -from ._preflight import describe_auth_sources -from ._tool_converter import ( - build_copilot_tools, - convert_tools_to_openai_format, - deep_decode_json_strings, -) - -logger = logging.getLogger(__name__) - - -class ChatCopilot(BaseChatModel): - """LangChain chat model backed by the GitHub Copilot SDK. - - Example: - >>> from copilot_langchain import ChatCopilot - >>> - >>> llm = ChatCopilot(model="gpt-4.1") - >>> response = await llm.ainvoke("Hello, how are you?") - >>> print(response.content) - - With tools: - >>> from langchain_core.tools import tool - >>> - >>> @tool - >>> def add(a: int, b: int) -> int: - ... \"\"\"Add two numbers.\"\"\" - ... return a + b - >>> - >>> llm_with_tools = ChatCopilot(model="gpt-4.1").bind_tools([add]) - >>> response = await llm_with_tools.ainvoke("What is 2 + 3?") - - With structured output: - >>> from pydantic import BaseModel - >>> - >>> class Answer(BaseModel): - ... value: int - ... explanation: str - >>> - >>> chain = ChatCopilot(model="gpt-4.1").with_structured_output(Answer) - >>> result = await chain.ainvoke("What is 2 + 2?") - >>> print(result.value) - """ - - model: str = "gpt-4.1" - """Model identifier to use (e.g. 'gpt-4.1', 'claude-sonnet-4').""" - - timeout: float = 120.0 - """Timeout in seconds for waiting on a response.""" - - copilot_client_options: dict[str, Any] = Field(default_factory=dict) - """Options passed to CopilotClient() constructor.""" - - # Private attributes (not serialised by Pydantic) - _manager: CopilotClientManager = PrivateAttr(default=None) - _bound_tools: list[dict[str, Any]] = PrivateAttr(default_factory=list) - _tool_choice: Optional[str] = PrivateAttr(default=None) - _ls_structured_output_format: Optional[dict[str, Any]] = PrivateAttr(default=None) - - def model_post_init(self, __context: Any) -> None: - self._manager = CopilotClientManager(self.copilot_client_options) - - # ------------------------------------------------------------------ # - # LangChain required properties - # ------------------------------------------------------------------ # - - @property - def _llm_type(self) -> str: - return "copilot-sdk" - - @property - def _identifying_params(self) -> dict[str, Any]: - return {"model": self.model} - - # ------------------------------------------------------------------ # - # Lifecycle - # ------------------------------------------------------------------ # - - async def aclose(self) -> None: - """Shut down the underlying Copilot CLI process.""" - await self._manager.close() - - # ------------------------------------------------------------------ # - # Structured output (JSON-based, bypasses tool calling) - # ------------------------------------------------------------------ # - - def with_structured_output( - self, - schema: Any, - *, - include_raw: bool = False, - **kwargs: Any, - ) -> Any: - """Return a chain that produces structured output via JSON text parsing. - - The Copilot CLI's model ignores tool-calling instructions and produces - natural language responses even when tools are registered. This override - injects a JSON schema requirement directly into the system prompt and - parses the model's text response, which is far more reliable. - """ - from pydantic import BaseModel as PydanticBaseModel - - from langchain_core.runnables import RunnableLambda - - is_pydantic = isinstance(schema, type) and issubclass(schema, PydanticBaseModel) - schema_json = schema.model_json_schema() if is_pydantic else schema - schema_str = json.dumps(schema_json, indent=2) - - json_instruction = ( - "\n\n# CRITICAL OUTPUT FORMAT REQUIREMENT\n" - "You MUST respond with ONLY a valid JSON object. No prose, no markdown, " - "no explanations, no code fences.\n" - "Your ENTIRE response must be a single valid JSON object matching this schema:\n" - f"{schema_str}\n" - "Start your response immediately with `{` and end with `}`." - ) - - def _inject(messages: list[BaseMessage]) -> list[BaseMessage]: - out: list[BaseMessage] = [] - injected = False - for msg in messages: - if isinstance(msg, SystemMessage) and not injected: - out.append(SystemMessage(content=msg.content + json_instruction)) - injected = True - else: - out.append(msg) - if not injected: - out.insert(0, SystemMessage(content=json_instruction.lstrip())) - return out - - def _parse(ai_message: AIMessage) -> Any: - content = (ai_message.content or "").strip() - # Strip markdown code fences if present - if content.startswith("```"): - lines = content.split("\n") - content = "\n".join(lines[1:-1]).strip() - # Extract outermost JSON object - start = content.find("{") - end = content.rfind("}") + 1 - if start == -1 or end == 0: - raise ValueError( - f"No JSON object found in model response.\n" - f"--- LLM output ---\n{content}\n--- end ---" - ) - json_text = content[start:end] - try: - parsed = json.loads(json_text) - except json.JSONDecodeError as exc: - raise ValueError( - f"Model returned invalid JSON: {exc}\n" - f"--- LLM output ---\n{content}\n--- end ---" - ) from exc - if not is_pydantic: - return parsed - try: - return schema.model_validate(parsed) - except Exception as exc: - raise ValueError( - f"Model output did not match the expected schema: {exc}\n" - f"--- LLM output ---\n{content}\n--- end ---" - ) from exc - - chain = RunnableLambda(_inject) | self | RunnableLambda(_parse) - return chain - - # ------------------------------------------------------------------ # - # Tool binding - # ------------------------------------------------------------------ # - - def bind_tools( - self, - tools: Sequence[dict[str, Any] | type | Callable | BaseTool], - *, - tool_choice: str | None = None, - **kwargs: Any, - ) -> ChatCopilot: - """Return a new ChatCopilot with tools bound. - - Args: - tools: Tools to make available to the model. - tool_choice: When set to "any", forces the model to use a tool. - Used internally by with_structured_output(). - - Returns: - A new ChatCopilot instance with the tools bound. - """ - openai_tools = convert_tools_to_openai_format(tools) - new = self.model_copy() - new._bound_tools = openai_tools - new._tool_choice = tool_choice - new._ls_structured_output_format = kwargs.get("ls_structured_output_format") - # Share the same client manager so the subprocess is not restarted - new._manager = self._manager - return new - - # ------------------------------------------------------------------ # - # Core generation (async — the native path) - # ------------------------------------------------------------------ # - - async def _agenerate( - self, - messages: list[BaseMessage], - stop: list[str] | None = None, - run_manager: AsyncCallbackManagerForLLMRun | None = None, - **kwargs: Any, - ) -> ChatResult: - try: - client = await self._manager.ensure_client() - except CopilotSetupError: - raise - except Exception as exc: - raise CopilotSetupError( - f"Unexpected error initialising Copilot SDK: {type(exc).__name__}: {exc}\n\n" - + describe_auth_sources() - ) from exc - - # Build session config - session_config: SessionConfig = { - "model": kwargs.get("model", self.model), - "available_tools": [], # Disable built-in tools - } - - # Merge any extra tools from kwargs with bound tools - extra_tools = kwargs.get("tools", []) - all_openai_tools = self._bound_tools + ( - convert_tools_to_openai_format(extra_tools) if extra_tools else [] - ) - if all_openai_tools: - session_config["tools"] = build_copilot_tools(all_openai_tools) - - # System message - system_content = extract_system_message(messages) - if system_content: - base_system = system_content - prompt_messages = [m for m in messages if not isinstance(m, SystemMessage)] - else: - base_system = "You are a helpful assistant." - prompt_messages = messages - - session_config["system_message"] = {"mode": "replace", "content": base_system} - session_config["infinite_sessions"] = {"enabled": False} - - session = await client.create_session( - on_permission_request=PermissionHandler.approve_all, - **session_config, - ) - try: - prompt = messages_to_prompt(prompt_messages) - - tool_requests: list[Any] = [] - - def _event_handler(event: Any) -> None: - # New SDK (protocol v3): custom tool calls come as ExternalToolRequestedData - # broadcast events rather than AssistantMessageData.tool_requests. - if isinstance(event.data, ExternalToolRequestedData): - tool_requests.append(event.data) - return - # Legacy fallback: tool calls in AssistantMessageData.tool_requests - if event.type == SessionEventType.ASSISTANT_MESSAGE: - if event.data.tool_requests: - tool_requests.extend(event.data.tool_requests) - - unsubscribe = session.on(_event_handler) - try: - response = await session.send_and_wait( - prompt, - timeout=self.timeout, - ) - finally: - unsubscribe() - - content = "" - if response and response.data and response.data.content: - content = response.data.content - - if response and response.data and response.data.tool_requests: - for tr in response.data.tool_requests: - if tr not in tool_requests: - tool_requests.append(tr) - - tool_calls = [] - seen_ids: set[str] = set() - for tr in tool_requests: - if isinstance(tr, ExternalToolRequestedData): - name, call_id, args = tr.tool_name, tr.tool_call_id, tr.arguments - else: - name, call_id, args = tr.name, tr.tool_call_id, tr.arguments - if call_id in seen_ids: - continue - seen_ids.add(call_id) - if isinstance(args, str): - try: - args = json.loads(args) - except (json.JSONDecodeError, TypeError): - args = {"raw": args} - elif args is None: - args = {} - if isinstance(args, dict): - args = deep_decode_json_strings(args) - tool_calls.append( - { - "name": name, - "args": args if isinstance(args, dict) else {"raw": args}, - "id": call_id, - } - ) - - ai_message = AIMessage( - content=content, - tool_calls=tool_calls if tool_calls else [], - response_metadata={"model": self.model}, - ) - return ChatResult(generations=[ChatGeneration(message=ai_message)]) - finally: - await session.destroy() - - # ------------------------------------------------------------------ # - # Sync generation (bridges to async) - # ------------------------------------------------------------------ # - - def _generate( - self, - messages: list[BaseMessage], - stop: list[str] | None = None, - run_manager: CallbackManagerForLLMRun | None = None, - **kwargs: Any, - ) -> ChatResult: - """Synchronous generation — delegates to the async implementation.""" - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = None - - if loop and loop.is_running(): - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit( - asyncio.run, - self._agenerate(messages, stop, None, **kwargs), - ) - return future.result() - else: - return asyncio.run(self._agenerate(messages, stop, None, **kwargs))