diff --git a/evaluation/README.md b/evaluation/README.md index b78c1c941..49d67c955 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -181,6 +181,52 @@ uv run --project evaluation powercontext-eval swebench-pro create-batch \ --idempotency-key "stability-$(date -u +%Y%m%dT%H%M%SZ)" ``` +## LongMemEval-V2 smoke input validation + +The LongMemEval-V2 command currently validates a fixed smoke subset and writes +its preflight artifacts. It does not yet invoke a PowerContext Memory adapter, +a Reader, or a Judge. + +Prepare a detached upstream checkout at the pinned harness commit and download +the matching LongMemEval-V2 data root outside this repository. The checked-in +small-tier lock and smoke manifest fix the data revision, three core data-file +hashes, ten source-ordered questions, all five published abilities, and both +published domains. The dataset lock has this shape: + +```json +{ + "schema": "powercontext.longmemeval-v2-dataset-lock.v1", + "upstream": { + "repository": "https://github.com/xiaowu0162/LongMemEval-V2", + "harness_commit": "2cc8c540bdb87fe6761629b585e727e1c4704520" + }, + "dataset_revision": "DATASET_REVISION", + "tier": "small", + "files": { + "questions.jsonl": "SHA256", + "trajectories.jsonl": "SHA256", + "haystacks/lme_v2_small.json": "SHA256" + } +} +``` + +The smoke manifest fixes question IDs, their upstream source order, and coverage +of the published ability categories. Run the preflight against a new output +directory: + +```bash +uv run --project evaluation powercontext-eval longmemeval-v2 smoke \ + --harness-root /path/to/LongMemEval-V2 \ + --data-root /path/to/longmemeval-v2-data \ + --dataset-lock evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json \ + --smoke-manifest evaluation/locks/longmemeval-v2-small-v1.smoke.json \ + --output-dir /path/to/new-smoke-artifacts +``` + +The command refuses a harness checkout at a different commit, mismatched input +hashes, invalid or incomplete smoke coverage, and an existing output directory. +It writes `manifest.json` and `subset.json`, both labelled as a smoke subset. + ## Configuration files Only the environment file configures the evaluation platform. The other files either belong to Codex or are diff --git a/evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json b/evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json new file mode 100644 index 000000000..59532327b --- /dev/null +++ b/evaluation/locks/longmemeval-v2-small-v1.dataset-lock.json @@ -0,0 +1,14 @@ +{ + "schema": "powercontext.longmemeval-v2-dataset-lock.v1", + "upstream": { + "repository": "https://github.com/xiaowu0162/LongMemEval-V2", + "harness_commit": "2cc8c540bdb87fe6761629b585e727e1c4704520" + }, + "dataset_revision": "f152293e235517d504809563c833d7190b8c713b", + "tier": "small", + "files": { + "questions.jsonl": "0a3ae5ebea938c24d7800e1e0b0828e08ae1646f939a53853b2b8cdc08e292b7", + "trajectories.jsonl": "363cec9a8e87aa8d9101ce4e600aadbf7031d674056ebe4f969e8424abc5f3c6", + "haystacks/lme_v2_small.json": "9b5301defb23a088a5f06e45ff8d5f35e569d78305a66d492046a9fff9b46593" + } +} diff --git a/evaluation/locks/longmemeval-v2-small-v1.smoke.json b/evaluation/locks/longmemeval-v2-small-v1.smoke.json new file mode 100644 index 000000000..f2eee75d9 --- /dev/null +++ b/evaluation/locks/longmemeval-v2-small-v1.smoke.json @@ -0,0 +1,46 @@ +{ + "schema": "powercontext.longmemeval-v2-smoke.v1", + "tier": "small", + "cases": [ + { + "question_id": "01307e07", + "ability": "dynamic_state" + }, + { + "question_id": "025db8ef", + "ability": "workflow_knowledge" + }, + { + "question_id": "057a2d4d", + "ability": "static_state" + }, + { + "question_id": "059974dd", + "ability": "premise_awareness" + }, + { + "question_id": "18b91103", + "ability": "environment_gotchas" + }, + { + "question_id": "00aa905a", + "ability": "dynamic_state" + }, + { + "question_id": "01f6e679", + "ability": "premise_awareness" + }, + { + "question_id": "0574c69a", + "ability": "environment_gotchas" + }, + { + "question_id": "05cce9b3", + "ability": "static_state" + }, + { + "question_id": "07ab3723", + "ability": "workflow_knowledge" + } + ] +} diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/__init__.py b/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/__init__.py new file mode 100644 index 000000000..50d2fb26d --- /dev/null +++ b/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pinned LongMemEval-V2 input validation and smoke preparation.""" diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/catalog.py b/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/catalog.py new file mode 100644 index 000000000..6359d016b --- /dev/null +++ b/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/catalog.py @@ -0,0 +1,434 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fail-closed catalog for a pinned LongMemEval-V2 dataset layout.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Literal, TypeAlias + +from powercontext_eval.errors import PowerContextEvalError + +UPSTREAM_REPOSITORY = "https://github.com/xiaowu0162/LongMemEval-V2" +UPSTREAM_HARNESS_COMMIT = "2cc8c540bdb87fe6761629b585e727e1c4704520" +SMOKE_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-smoke.v1" +DATASET_LOCK_SCHEMA = "powercontext.longmemeval-v2-dataset-lock.v1" +RUN_INPUT_MANIFEST_SCHEMA = "powercontext.longmemeval-v2-run-input.v1" + +Tier: TypeAlias = Literal["small", "medium"] +Ability: TypeAlias = Literal[ + "static_state", + "dynamic_state", + "workflow_knowledge", + "environment_gotchas", + "premise_awareness", +] + +_ABILITIES: frozenset[Ability] = frozenset( + { + "static_state", + "dynamic_state", + "workflow_knowledge", + "environment_gotchas", + "premise_awareness", + } +) +_QUESTION_TYPE_ABILITIES: Mapping[str, Ability] = MappingProxyType( + { + "static-environment": "static_state", + "static-environment-abs": "premise_awareness", + "dynamic-environment": "dynamic_state", + "dynamic-environment-abs": "premise_awareness", + "procedure": "workflow_knowledge", + "procedure-abs": "premise_awareness", + "errors-gotchas": "environment_gotchas", + } +) + + +class LongMemEvalV2CatalogError(PowerContextEvalError): + """The pinned LongMemEval-V2 inputs cannot be trusted.""" + + +@dataclass(frozen=True) +class Question: + """The query-visible fields required to validate one upstream question.""" + + question_id: str + domain: Literal["web", "enterprise"] + question_type: str + + @property + def ability(self) -> Ability: + return _QUESTION_TYPE_ABILITIES[self.question_type] + + +@dataclass(frozen=True) +class SmokeCase: + """One fixed question selected for the bounded smoke workload.""" + + question_id: str + ability: Ability + + +@dataclass(frozen=True) +class DatasetLock: + """The immutable identity and file digests for one upstream data tier.""" + + tier: Tier + dataset_revision: str + file_digests: Mapping[str, str] + + def as_json(self) -> dict[str, object]: + return { + "schema": DATASET_LOCK_SCHEMA, + "upstream": { + "harness_commit": UPSTREAM_HARNESS_COMMIT, + "repository": UPSTREAM_REPOSITORY, + }, + "dataset_revision": self.dataset_revision, + "tier": self.tier, + "files": dict(self.file_digests), + } + + +@dataclass(frozen=True) +class SmokeSelection: + """A validated fixed subset for one LongMemEval-V2 tier.""" + + tier: Tier + cases: tuple[SmokeCase, ...] + + def as_json(self) -> dict[str, object]: + return { + "schema": SMOKE_MANIFEST_SCHEMA, + "tier": self.tier, + "cases": [{"question_id": case.question_id, "ability": case.ability} for case in self.cases], + } + + +@dataclass(frozen=True) +class LongMemEvalV2Catalog: + """Validated LongMemEval-V2 input identities for one upstream data root.""" + + data_root: Path + tier: Tier + input_digests: Mapping[str, str] + questions: Mapping[str, Question] + source_question_ids: tuple[str, ...] + + @classmethod + def load( + cls, + data_root: Path, + *, + tier: Tier, + expected_digests: Mapping[str, str] | None = None, + ) -> LongMemEvalV2Catalog: + """Load and validate the three upstream files required by one tier.""" + + if tier not in {"small", "medium"}: + raise LongMemEvalV2CatalogError(f"Unsupported LongMemEval-V2 tier: {tier}") + root = data_root.resolve() + paths = { + "questions.jsonl": root / "questions.jsonl", + "trajectories.jsonl": root / "trajectories.jsonl", + f"haystacks/lme_v2_{tier}.json": root / "haystacks" / f"lme_v2_{tier}.json", + } + digests = MappingProxyType({name: _file_digest(path, name) for name, path in paths.items()}) + _validate_expected_digests(digests, expected_digests) + + questions, source_question_ids = _questions(paths["questions.jsonl"]) + trajectories = _trajectories(paths["trajectories.jsonl"]) + haystack = _haystack(paths[f"haystacks/lme_v2_{tier}.json"]) + _validate_haystack(questions, trajectories, haystack) + return cls( + data_root=root, + tier=tier, + input_digests=digests, + questions=MappingProxyType(questions), + source_question_ids=source_question_ids, + ) + + def select_smoke(self, cases: Sequence[SmokeCase]) -> SmokeSelection: + """Validate a fixed smoke subset without deriving one from benchmark metadata.""" + + if not cases: + raise LongMemEvalV2CatalogError("Smoke subset must contain at least one question") + selected: list[SmokeCase] = [] + seen: set[str] = set() + source_positions = {question_id: index for index, question_id in enumerate(self.source_question_ids)} + for case in cases: + if case.question_id in seen: + raise LongMemEvalV2CatalogError(f"Smoke subset contains duplicate question id: {case.question_id}") + seen.add(case.question_id) + question = self.questions.get(case.question_id) + if question is None: + raise LongMemEvalV2CatalogError(f"Smoke subset references an unknown question id: {case.question_id}") + if question.ability != case.ability: + raise LongMemEvalV2CatalogError( + f"Smoke subset ability mismatch for {case.question_id}: " + f"expected {question.ability}, got {case.ability}" + ) + selected.append(case) + if [source_positions[case.question_id] for case in selected] != sorted( + source_positions[case.question_id] for case in selected + ): + raise LongMemEvalV2CatalogError("Smoke subset question order must match the upstream source order") + selected_abilities = {case.ability for case in selected} + missing = sorted(_ABILITIES - selected_abilities) + if missing: + raise LongMemEvalV2CatalogError(f"Smoke subset is missing published abilities: {', '.join(missing)}") + return SmokeSelection(tier=self.tier, cases=tuple(selected)) + + +def load_smoke_manifest(path: Path) -> SmokeSelection: + """Load one exact smoke-subset declaration without selecting questions dynamically.""" + + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise LongMemEvalV2CatalogError(f"Cannot read LongMemEval-V2 smoke manifest: {path}") from error + if not isinstance(value, dict) or set(value) != {"schema", "tier", "cases"}: + raise LongMemEvalV2CatalogError("Smoke manifest must contain only schema, tier, and cases") + if value["schema"] != SMOKE_MANIFEST_SCHEMA: + raise LongMemEvalV2CatalogError("Smoke manifest schema is unsupported") + tier = value["tier"] + if tier not in {"small", "medium"}: + raise LongMemEvalV2CatalogError("Smoke manifest tier must be small or medium") + raw_cases = value["cases"] + if not isinstance(raw_cases, list): + raise LongMemEvalV2CatalogError("Smoke manifest cases must be an array") + cases: list[SmokeCase] = [] + for index, raw_case in enumerate(raw_cases): + if not isinstance(raw_case, dict) or set(raw_case) != {"question_id", "ability"}: + raise LongMemEvalV2CatalogError(f"Smoke manifest case {index} has an invalid shape") + question_id = raw_case["question_id"] + ability = raw_case["ability"] + if not isinstance(question_id, str) or not question_id.strip(): + raise LongMemEvalV2CatalogError(f"Smoke manifest case {index} has an invalid question id") + if ability not in _ABILITIES: + raise LongMemEvalV2CatalogError(f"Smoke manifest case {index} has an invalid ability") + cases.append(SmokeCase(question_id=question_id, ability=ability)) + return SmokeSelection(tier=tier, cases=tuple(cases)) + + +def load_dataset_lock(path: Path) -> DatasetLock: + """Load a fixed input identity before reading any benchmark data.""" + + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise LongMemEvalV2CatalogError(f"Cannot read LongMemEval-V2 dataset lock: {path}") from error + if not isinstance(value, dict) or set(value) != {"schema", "upstream", "dataset_revision", "tier", "files"}: + raise LongMemEvalV2CatalogError( + "Dataset lock must contain only schema, upstream, dataset_revision, tier, and files" + ) + if value["schema"] != DATASET_LOCK_SCHEMA: + raise LongMemEvalV2CatalogError("Dataset lock schema is unsupported") + upstream = value["upstream"] + if not isinstance(upstream, dict) or set(upstream) != {"repository", "harness_commit"}: + raise LongMemEvalV2CatalogError("Dataset lock upstream identity is invalid") + if upstream["repository"] != UPSTREAM_REPOSITORY or upstream["harness_commit"] != UPSTREAM_HARNESS_COMMIT: + raise LongMemEvalV2CatalogError("Dataset lock does not match the pinned LongMemEval-V2 harness") + dataset_revision = value["dataset_revision"] + if not isinstance(dataset_revision, str) or not dataset_revision.strip(): + raise LongMemEvalV2CatalogError("Dataset lock dataset revision must be non-empty") + tier = value["tier"] + if tier not in {"small", "medium"}: + raise LongMemEvalV2CatalogError("Dataset lock tier must be small or medium") + files = value["files"] + expected_names = { + "questions.jsonl", + "trajectories.jsonl", + f"haystacks/lme_v2_{tier}.json", + } + if not isinstance(files, dict) or set(files) != expected_names: + raise LongMemEvalV2CatalogError("Dataset lock must provide digests for exactly the required files") + if any(not isinstance(digest, str) or len(digest) != 64 or not _is_lower_hex(digest) for digest in files.values()): + raise LongMemEvalV2CatalogError("Dataset lock file digests must be lowercase SHA-256 values") + return DatasetLock(tier=tier, dataset_revision=dataset_revision, file_digests=MappingProxyType(dict(files))) + + +def validate_harness_checkout(harness_root: Path) -> None: + """Require the configured upstream checkout to be exactly the pinned harness commit.""" + + harness = harness_root.resolve() + if not (harness / "evaluation" / "harness.py").is_file(): + raise LongMemEvalV2CatalogError(f"LongMemEval-V2 harness is missing evaluation/harness.py: {harness}") + try: + completed = subprocess.run( + ["git", "-C", str(harness), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as error: + raise LongMemEvalV2CatalogError(f"Cannot inspect LongMemEval-V2 harness checkout: {harness}") from error + revision = completed.stdout.strip() + if completed.returncode != 0 or revision != UPSTREAM_HARNESS_COMMIT: + raise LongMemEvalV2CatalogError( + f"LongMemEval-V2 harness checkout must be {UPSTREAM_HARNESS_COMMIT}, got {revision or 'unknown'}" + ) + + +def _file_digest(path: Path, label: str) -> str: + """Hash one input without materializing large trajectory files in memory.""" + + hasher = hashlib.sha256() + bytes_read = 0 + try: + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + hasher.update(chunk) + bytes_read += len(chunk) + except OSError as error: + raise LongMemEvalV2CatalogError(f"Missing LongMemEval-V2 input {label}: {path}") from error + if bytes_read == 0: + raise LongMemEvalV2CatalogError(f"LongMemEval-V2 input {label} is blank") + return hasher.hexdigest() + + +def _validate_expected_digests(actual: Mapping[str, str], expected: Mapping[str, str] | None) -> None: + if expected is None: + return + if set(expected) != set(actual): + raise LongMemEvalV2CatalogError("Expected input digests must name exactly the required files") + for name, digest in actual.items(): + if expected[name] != digest: + raise LongMemEvalV2CatalogError(f"LongMemEval-V2 SHA-256 mismatch for {name}") + + +def _questions(path: Path) -> tuple[dict[str, Question], tuple[str, ...]]: + questions: dict[str, Question] = {} + source_order: list[str] = [] + for index, row in enumerate(_jsonl(path, "questions.jsonl")): + question_id = _nonblank(row.get("id"), f"question {index} id") + if question_id in questions: + raise LongMemEvalV2CatalogError(f"Duplicate LongMemEval-V2 question id: {question_id}") + domain = row.get("domain") + if domain not in {"web", "enterprise"}: + raise LongMemEvalV2CatalogError(f"Invalid question domain for {question_id}") + _nonblank(row.get("question"), f"question text for {question_id}") + question_type = row.get("question_type") + if question_type not in _QUESTION_TYPE_ABILITIES: + raise LongMemEvalV2CatalogError(f"Unsupported question type for {question_id}") + questions[question_id] = Question(question_id=question_id, domain=domain, question_type=question_type) + source_order.append(question_id) + return questions, tuple(source_order) + + +def _trajectories(path: Path) -> Mapping[str, Literal["web", "enterprise"]]: + trajectories: dict[str, Literal["web", "enterprise"]] = {} + for index, row in enumerate(_jsonl(path, "trajectories.jsonl")): + trajectory_id = _nonblank(row.get("id"), f"trajectory {index} id") + if trajectory_id in trajectories: + raise LongMemEvalV2CatalogError(f"Duplicate LongMemEval-V2 trajectory id: {trajectory_id}") + domain = row.get("domain") + if domain not in {"web", "enterprise"}: + raise LongMemEvalV2CatalogError(f"Invalid trajectory domain for {trajectory_id}") + trajectories[trajectory_id] = domain + return MappingProxyType(trajectories) + + +def _haystack(path: Path) -> Mapping[str, tuple[str, ...]]: + try: + with path.open(encoding="utf-8") as source: + value = json.load(source) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise LongMemEvalV2CatalogError("LongMemEval-V2 haystack is not valid JSON") from error + if not isinstance(value, dict): + raise LongMemEvalV2CatalogError("LongMemEval-V2 haystack must be a JSON object") + haystack: dict[str, tuple[str, ...]] = {} + for question_id, trajectory_ids in value.items(): + if not isinstance(question_id, str) or not question_id: + raise LongMemEvalV2CatalogError("LongMemEval-V2 haystack contains an invalid question id") + if not isinstance(trajectory_ids, list) or not trajectory_ids: + raise LongMemEvalV2CatalogError(f"LongMemEval-V2 haystack is empty for {question_id}") + if not all(isinstance(trajectory_id, str) and trajectory_id for trajectory_id in trajectory_ids): + raise LongMemEvalV2CatalogError( + f"LongMemEval-V2 haystack contains an invalid trajectory id for {question_id}" + ) + if len(trajectory_ids) != len(set(trajectory_ids)): + raise LongMemEvalV2CatalogError( + f"LongMemEval-V2 haystack contains duplicate trajectories for {question_id}" + ) + haystack[question_id] = tuple(trajectory_ids) + return MappingProxyType(haystack) + + +def _validate_haystack( + questions: Mapping[str, Question], + trajectories: Mapping[str, Literal["web", "enterprise"]], + haystack: Mapping[str, tuple[str, ...]], +) -> None: + if set(haystack) != set(questions): + raise LongMemEvalV2CatalogError("LongMemEval-V2 questions and haystack ids must match exactly") + for question_id, trajectory_ids in haystack.items(): + question = questions[question_id] + for trajectory_id in trajectory_ids: + trajectory_domain = trajectories.get(trajectory_id) + if trajectory_domain is None: + raise LongMemEvalV2CatalogError( + f"LongMemEval-V2 haystack references an unknown trajectory: {trajectory_id}" + ) + if trajectory_domain != question.domain: + raise LongMemEvalV2CatalogError(f"LongMemEval-V2 haystack crosses domains for question {question_id}") + + +def _jsonl(path: Path, label: str) -> Iterator[dict[str, object]]: + """Parse one JSONL input incrementally to support multi-gigabyte trajectories.""" + + found = False + try: + with path.open(encoding="utf-8") as source: + for line_number, line in enumerate(source, start=1): + if not line.strip(): + continue + found = True + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise LongMemEvalV2CatalogError( + f"LongMemEval-V2 input {label} has invalid JSON at {line_number}" + ) from error + if not isinstance(value, dict): + raise LongMemEvalV2CatalogError( + f"LongMemEval-V2 input {label} has a non-object row at {line_number}" + ) + yield value + except (OSError, UnicodeDecodeError) as error: + raise LongMemEvalV2CatalogError(f"Cannot read LongMemEval-V2 input {label}: {path}") from error + if not found: + raise LongMemEvalV2CatalogError(f"LongMemEval-V2 input {label} is blank") + + +def _nonblank(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise LongMemEvalV2CatalogError(f"Invalid LongMemEval-V2 {label}") + return value + + +def _is_lower_hex(value: str) -> bool: + return all(character in "0123456789abcdef" for character in value) diff --git a/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/smoke.py b/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/smoke.py new file mode 100644 index 000000000..8f5f66dfc --- /dev/null +++ b/evaluation/src/powercontext_eval/benchmarks/longmemeval_v2/smoke.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create the bounded, model-free input artifacts for a LongMemEval-V2 smoke run.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path + +from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( + RUN_INPUT_MANIFEST_SCHEMA, + UPSTREAM_HARNESS_COMMIT, + UPSTREAM_REPOSITORY, + LongMemEvalV2Catalog, + LongMemEvalV2CatalogError, + load_dataset_lock, + load_smoke_manifest, + validate_harness_checkout, +) + + +@dataclass(frozen=True) +class PreparedSmokeRun: + """The inspectable artifacts emitted before model-backed evaluation begins.""" + + output_dir: Path + manifest_path: Path + subset_path: Path + + +def prepare_smoke_run( + *, + data_root: Path, + dataset_lock: Path, + harness_root: Path, + smoke_manifest: Path, + output_dir: Path, +) -> PreparedSmokeRun: + """Validate one fixed subset and create non-overwritable smoke input artifacts.""" + + lock = load_dataset_lock(dataset_lock) + selection = load_smoke_manifest(smoke_manifest) + if selection.tier != lock.tier: + raise LongMemEvalV2CatalogError("Smoke manifest tier does not match the dataset lock") + validate_harness_checkout(harness_root) + catalog = LongMemEvalV2Catalog.load(data_root, tier=selection.tier, expected_digests=lock.file_digests) + catalog.select_smoke(selection.cases) + try: + output_dir.mkdir(parents=True, exist_ok=False) + except FileExistsError as error: + raise LongMemEvalV2CatalogError(f"Refusing to overwrite smoke artifacts: {output_dir}") from error + + subset_path = output_dir / "subset.json" + manifest_path = output_dir / "manifest.json" + subset = selection.as_json() + _write_json(subset_path, subset) + _write_json( + manifest_path, + { + "schema": RUN_INPUT_MANIFEST_SCHEMA, + "classification": "smoke-subset", + "upstream": { + "harness_commit": UPSTREAM_HARNESS_COMMIT, + "repository": UPSTREAM_REPOSITORY, + }, + "dataset": { + "files": dict(catalog.input_digests), + "revision": lock.dataset_revision, + "tier": catalog.tier, + }, + "dataset_lock": {"content_sha256": hashlib.sha256(dataset_lock.read_bytes()).hexdigest()}, + "smoke_manifest": { + "content_sha256": hashlib.sha256(smoke_manifest.read_bytes()).hexdigest(), + "subset_sha256": hashlib.sha256(subset_path.read_bytes()).hexdigest(), + }, + }, + ) + return PreparedSmokeRun(output_dir=output_dir, manifest_path=manifest_path, subset_path=subset_path) + + +def _write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/evaluation/src/powercontext_eval/cli.py b/evaluation/src/powercontext_eval/cli.py index ee1eedee3..6d06224f9 100644 --- a/evaluation/src/powercontext_eval/cli.py +++ b/evaluation/src/powercontext_eval/cli.py @@ -32,6 +32,8 @@ import typer from pydantic import ValidationError +from powercontext_eval.benchmarks.longmemeval_v2.catalog import LongMemEvalV2CatalogError +from powercontext_eval.benchmarks.longmemeval_v2.smoke import prepare_smoke_run from powercontext_eval.benchmarks.swebench_pro.catalog import PUBLIC_V2_TASK_SET, SweBenchProCatalog, TaskSet from powercontext_eval.codex import DEFAULT_CODEX_MODEL, DEFAULT_REASONING_EFFORT from powercontext_eval.models import TreatmentMode @@ -44,7 +46,9 @@ app = typer.Typer(no_args_is_help=True, help="PowerContext evaluation runner.") swebench_pro_app = typer.Typer(no_args_is_help=True, help="Pinned SWE-bench Pro evaluation.") +longmemeval_v2_app = typer.Typer(no_args_is_help=True, help="Pinned LongMemEval-V2 evaluation.") app.add_typer(swebench_pro_app, name="swebench-pro") +app.add_typer(longmemeval_v2_app, name="longmemeval-v2") @app.callback() @@ -169,6 +173,39 @@ def codex_contract_smoke( typer.echo(json.dumps(outcome, ensure_ascii=False, sort_keys=True)) +@longmemeval_v2_app.command("smoke") +def longmemeval_v2_smoke( + data_root: Path = typer.Option(..., "--data-root"), + dataset_lock: Path = typer.Option(..., "--dataset-lock"), + harness_root: Path = typer.Option(..., "--harness-root"), + smoke_manifest: Path = typer.Option(..., "--smoke-manifest"), + output_dir: Path = typer.Option(..., "--output-dir"), +) -> None: + """Validate fixed LongMemEval-V2 inputs and write smoke artifacts without calling a model.""" + + try: + prepared = prepare_smoke_run( + data_root=data_root, + dataset_lock=dataset_lock, + harness_root=harness_root, + smoke_manifest=smoke_manifest, + output_dir=output_dir, + ) + except LongMemEvalV2CatalogError as error: + raise typer.BadParameter(str(error)) from None + typer.echo( + json.dumps( + { + "classification": "smoke-subset", + "manifest": str(prepared.manifest_path), + "subset": str(prepared.subset_path), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + @swebench_pro_app.command("run") def swebench_pro_run( root_path: str = typer.Option(..., "--root"), diff --git a/evaluation/tests/unit/test_cli.py b/evaluation/tests/unit/test_cli.py index 41622603b..5450a0642 100644 --- a/evaluation/tests/unit/test_cli.py +++ b/evaluation/tests/unit/test_cli.py @@ -25,6 +25,7 @@ from typer.testing import CliRunner from powercontext_eval.cli import app +from powercontext_eval.benchmarks.longmemeval_v2.smoke import PreparedSmokeRun from powercontext_eval.runner import MinimalRunResult, RunConfig @@ -36,6 +37,44 @@ def test_cli_help_describes_the_evaluation_runner() -> None: assert not isinstance(result.exception, RuntimeError) +def test_longmemeval_v2_smoke_prepares_input_artifacts_without_a_model(monkeypatch, tmp_path: Path) -> None: + def prepare(**kwargs: object) -> PreparedSmokeRun: + assert kwargs == { + "data_root": Path("/data"), + "dataset_lock": Path("/dataset-lock.json"), + "harness_root": Path("/harness"), + "smoke_manifest": Path("/smoke.json"), + "output_dir": Path("/output"), + } + return PreparedSmokeRun( + output_dir=tmp_path / "output", + manifest_path=tmp_path / "output" / "manifest.json", + subset_path=tmp_path / "output" / "subset.json", + ) + + monkeypatch.setattr("powercontext_eval.cli.prepare_smoke_run", prepare) + result = CliRunner().invoke( + app, + [ + "longmemeval-v2", + "smoke", + "--data-root", + "/data", + "--dataset-lock", + "/dataset-lock.json", + "--harness-root", + "/harness", + "--smoke-manifest", + "/smoke.json", + "--output-dir", + "/output", + ], + ) + + assert result.exit_code == 0, result.output + assert '"classification": "smoke-subset"' in result.output + + def test_codex_contract_smoke_is_an_executable_injectable_cli(monkeypatch) -> None: calls: list[dict[str, object]] = [] diff --git a/evaluation/tests/unit/test_longmemeval_v2_catalog.py b/evaluation/tests/unit/test_longmemeval_v2_catalog.py new file mode 100644 index 000000000..703ad3773 --- /dev/null +++ b/evaluation/tests/unit/test_longmemeval_v2_catalog.py @@ -0,0 +1,293 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +import pytest + +from powercontext_eval.benchmarks.longmemeval_v2.catalog import ( + RUN_INPUT_MANIFEST_SCHEMA, + SMOKE_MANIFEST_SCHEMA, + UPSTREAM_HARNESS_COMMIT, + LongMemEvalV2Catalog, + LongMemEvalV2CatalogError, + SmokeCase, + load_dataset_lock, + load_smoke_manifest, + validate_harness_checkout, +) +from powercontext_eval.benchmarks.longmemeval_v2.smoke import prepare_smoke_run + +LOCKS = Path(__file__).parents[2] / "locks" + + +def _write_jsonl(path: Path, values: list[dict[str, object]]) -> None: + path.write_text("".join(json.dumps(value) + "\n" for value in values), encoding="utf-8") + + +def data_root(tmp_path: Path) -> Path: + root = tmp_path / "data" + (root / "haystacks").mkdir(parents=True) + _write_jsonl( + root / "questions.jsonl", + [ + {"id": "q-static", "domain": "web", "question": "static", "question_type": "static-environment"}, + {"id": "q-dynamic", "domain": "web", "question": "dynamic", "question_type": "dynamic-environment"}, + {"id": "q-workflow", "domain": "enterprise", "question": "workflow", "question_type": "procedure"}, + {"id": "q-gotcha", "domain": "enterprise", "question": "gotcha", "question_type": "errors-gotchas"}, + {"id": "q-premise", "domain": "web", "question": "premise", "question_type": "static-environment-abs"}, + ], + ) + _write_jsonl( + root / "trajectories.jsonl", + [ + {"id": "t-web", "domain": "web"}, + {"id": "t-enterprise", "domain": "enterprise"}, + ], + ) + (root / "haystacks" / "lme_v2_small.json").write_text( + json.dumps( + { + "q-static": ["t-web"], + "q-dynamic": ["t-web"], + "q-workflow": ["t-enterprise"], + "q-gotcha": ["t-enterprise"], + "q-premise": ["t-web"], + } + ), + encoding="utf-8", + ) + return root + + +def smoke_manifest(path: Path) -> Path: + path.write_text( + json.dumps( + { + "schema": SMOKE_MANIFEST_SCHEMA, + "tier": "small", + "cases": [ + {"question_id": "q-static", "ability": "static_state"}, + {"question_id": "q-dynamic", "ability": "dynamic_state"}, + {"question_id": "q-workflow", "ability": "workflow_knowledge"}, + {"question_id": "q-gotcha", "ability": "environment_gotchas"}, + {"question_id": "q-premise", "ability": "premise_awareness"}, + ], + } + ), + encoding="utf-8", + ) + return path + + +def dataset_lock(path: Path, root: Path) -> Path: + files = { + "questions.jsonl": root / "questions.jsonl", + "trajectories.jsonl": root / "trajectories.jsonl", + "haystacks/lme_v2_small.json": root / "haystacks" / "lme_v2_small.json", + } + path.write_text( + json.dumps( + { + "schema": "powercontext.longmemeval-v2-dataset-lock.v1", + "upstream": { + "repository": "https://github.com/xiaowu0162/LongMemEval-V2", + "harness_commit": UPSTREAM_HARNESS_COMMIT, + }, + "dataset_revision": "fixture-data-revision", + "tier": "small", + "files": {name: hashlib.sha256(value.read_bytes()).hexdigest() for name, value in files.items()}, + } + ), + encoding="utf-8", + ) + return path + + +def test_catalog_validates_all_upstream_input_relationships(tmp_path: Path) -> None: + catalog = LongMemEvalV2Catalog.load(data_root(tmp_path), tier="small") + + assert catalog.source_question_ids == ("q-static", "q-dynamic", "q-workflow", "q-gotcha", "q-premise") + assert catalog.questions["q-workflow"].ability == "workflow_knowledge" + assert set(catalog.input_digests) == { + "questions.jsonl", + "trajectories.jsonl", + "haystacks/lme_v2_small.json", + } + + +def test_catalog_rejects_a_cross_domain_haystack(tmp_path: Path) -> None: + root = data_root(tmp_path) + (root / "haystacks" / "lme_v2_small.json").write_text( + json.dumps( + { + "q-static": ["t-enterprise"], + "q-dynamic": ["t-web"], + "q-workflow": ["t-enterprise"], + "q-gotcha": ["t-enterprise"], + "q-premise": ["t-web"], + } + ), + encoding="utf-8", + ) + + with pytest.raises(LongMemEvalV2CatalogError, match="crosses domains"): + LongMemEvalV2Catalog.load(root, tier="small") + + +def test_smoke_selection_requires_all_published_abilities(tmp_path: Path) -> None: + catalog = LongMemEvalV2Catalog.load(data_root(tmp_path), tier="small") + + with pytest.raises(LongMemEvalV2CatalogError, match="missing published abilities"): + catalog.select_smoke((SmokeCase(question_id="q-static", ability="static_state"),)) + + +def test_smoke_selection_requires_upstream_question_order(tmp_path: Path) -> None: + catalog = LongMemEvalV2Catalog.load(data_root(tmp_path), tier="small") + cases = load_smoke_manifest(smoke_manifest(tmp_path / "smoke.json")).cases + + with pytest.raises(LongMemEvalV2CatalogError, match="source order"): + catalog.select_smoke((cases[1], cases[0], *cases[2:])) + + +def test_catalog_skips_blank_jsonl_rows_like_the_upstream_loader(tmp_path: Path) -> None: + root = data_root(tmp_path) + questions = root / "questions.jsonl" + questions.write_text("\n" + questions.read_text(encoding="utf-8") + "\n", encoding="utf-8") + + catalog = LongMemEvalV2Catalog.load(root, tier="small") + + assert len(catalog.questions) == 5 + + +def test_catalog_streams_input_files_without_reading_them_all_at_once(monkeypatch, tmp_path: Path) -> None: + root = data_root(tmp_path) + + def read_bytes(*_args: object, **_kwargs: object) -> bytes: + raise AssertionError("catalog must stream large input files") + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + catalog = LongMemEvalV2Catalog.load(root, tier="small") + + assert len(catalog.questions) == 5 + + +def test_smoke_manifest_is_fixed_and_validated_against_the_catalog(tmp_path: Path) -> None: + root = data_root(tmp_path) + manifest = smoke_manifest(tmp_path / "smoke.json") + selection = load_smoke_manifest(manifest) + catalog = LongMemEvalV2Catalog.load(root, tier=selection.tier) + + assert catalog.select_smoke(selection.cases).as_json() == json.loads(manifest.read_text(encoding="utf-8")) + + +def test_dataset_lock_pins_exact_file_digests(tmp_path: Path) -> None: + root = data_root(tmp_path) + lock = dataset_lock(tmp_path / "dataset-lock.json", root) + loaded = load_dataset_lock(lock) + + LongMemEvalV2Catalog.load(root, tier=loaded.tier, expected_digests=loaded.file_digests) + (root / "questions.jsonl").write_text("{}\n", encoding="utf-8") + with pytest.raises(LongMemEvalV2CatalogError, match="SHA-256 mismatch"): + LongMemEvalV2Catalog.load(root, tier=loaded.tier, expected_digests=loaded.file_digests) + + +def test_checked_in_small_smoke_contract_is_valid() -> None: + lock = load_dataset_lock(LOCKS / "longmemeval-v2-small-v1.dataset-lock.json") + smoke = load_smoke_manifest(LOCKS / "longmemeval-v2-small-v1.smoke.json") + + assert lock.tier == "small" + assert smoke.tier == lock.tier + assert len(smoke.cases) == 10 + assert {case.ability for case in smoke.cases} == { + "dynamic_state", + "environment_gotchas", + "premise_awareness", + "static_state", + "workflow_knowledge", + } + + +def test_harness_checkout_requires_the_exact_pinned_revision(monkeypatch, tmp_path: Path) -> None: + harness = tmp_path / "harness" + (harness / "evaluation").mkdir(parents=True) + (harness / "evaluation" / "harness.py").write_text("", encoding="utf-8") + monkeypatch.setattr( + "powercontext_eval.benchmarks.longmemeval_v2.catalog.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, f"{UPSTREAM_HARNESS_COMMIT}\n", ""), + ) + + validate_harness_checkout(harness) + + +def test_harness_checkout_rejects_a_different_revision(monkeypatch, tmp_path: Path) -> None: + harness = tmp_path / "harness" + (harness / "evaluation").mkdir(parents=True) + (harness / "evaluation" / "harness.py").write_text("", encoding="utf-8") + monkeypatch.setattr( + "powercontext_eval.benchmarks.longmemeval_v2.catalog.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "0" * 40 + "\n", ""), + ) + + with pytest.raises(LongMemEvalV2CatalogError, match="harness checkout must be"): + validate_harness_checkout(harness) + + +def test_prepare_smoke_run_writes_non_overwritable_provenance(tmp_path: Path) -> None: + root = data_root(tmp_path) + smoke = smoke_manifest(tmp_path / "smoke.json") + lock = dataset_lock(tmp_path / "dataset-lock.json", root) + harness = tmp_path / "harness" + (harness / "evaluation").mkdir(parents=True) + (harness / "evaluation" / "harness.py").write_text("", encoding="utf-8") + output = tmp_path / "run" + + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr( + "powercontext_eval.benchmarks.longmemeval_v2.catalog.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, f"{UPSTREAM_HARNESS_COMMIT}\n", ""), + ) + + try: + prepare_smoke_run( + data_root=root, + dataset_lock=lock, + harness_root=harness, + smoke_manifest=smoke, + output_dir=output, + ) + manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) + assert manifest["schema"] == RUN_INPUT_MANIFEST_SCHEMA + assert manifest["classification"] == "smoke-subset" + assert manifest["upstream"]["harness_commit"] == UPSTREAM_HARNESS_COMMIT + assert manifest["dataset"]["revision"] == "fixture-data-revision" + assert manifest["dataset_lock"]["content_sha256"] == hashlib.sha256(lock.read_bytes()).hexdigest() + assert manifest["smoke_manifest"]["content_sha256"] == hashlib.sha256(smoke.read_bytes()).hexdigest() + assert "path_sha256" not in manifest["smoke_manifest"] + with pytest.raises(LongMemEvalV2CatalogError, match="Refusing to overwrite"): + prepare_smoke_run( + data_root=root, + dataset_lock=lock, + harness_root=harness, + smoke_manifest=smoke, + output_dir=output, + ) + finally: + monkeypatch.undo()