diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b9a372..206e2d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,4 +87,76 @@ jobs: run: | python -m pytest -v --cov=migra --cov-report=term-missing + characterize: + name: CLI Characterization + needs: lint + runs-on: ubuntu-latest + continue-on-error: true + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install system dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools + + - name: Install dependencies + run: | + pip install wheel pytest psycopg2-binary + + - name: Install MigraDiff with AI support + run: | + pip install -e ".[dev,ai]" + + - name: Create database user + run: | + psql -U postgres -h localhost -c "CREATE ROLE runner SUPERUSER LOGIN;" + psql -U postgres -h localhost -c "CREATE DATABASE runner OWNER runner;" + + - name: Phase A — capture current CLI behavior + env: + ANTHROPIC_API_KEY: sk-ant-char-test-dummy-key + run: | + python -m pytest tests/characterization/test_capture.py -v + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: characterization-${{ github.sha }} + path: tests/characterization/_artifacts/ + retention-days: 5 + + - name: Phase B — download base-branch artifact + id: download-base + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python scripts/download_base_artifact.py + + - name: Compare against base branch + if: steps.download-base.outcome == 'success' + run: | + python scripts/compare_characterization.py \ + tests/characterization/_artifacts \ + tests/characterization/_base_artifacts + diff --git a/.gitignore b/.gitignore index 631876b..9555245 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,8 @@ pip-wheel-metadata .vscode PROJECT_PLAN*.md + +# Characterization test artifacts (generated in CI, never committed) +tests/characterization/_artifacts/ +tests/characterization/_base_artifacts/ +tests/characterization/*_tmp/ diff --git a/scripts/compare_characterization.py b/scripts/compare_characterization.py new file mode 100644 index 0000000..745e62a --- /dev/null +++ b/scripts/compare_characterization.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +""" +Compare two directories of characterization artifacts and report drift. + +Usage: + python scripts/compare_characterization.py [--strict] + +Outputs: + - unified diff per scenario to stdout + - summary table to stdout + - summary table to $GITHUB_STEP_SUMMARY if set + +Exit codes: + 0 always (informational), unless --strict is passed + 1 any drift detected (only with --strict) +""" + +from __future__ import unicode_literals + +import difflib +import json +import os +import sys + + +def load_artifacts(artifact_dir): + result = {} + if not os.path.isdir(artifact_dir): + return result + for fname in os.listdir(artifact_dir): + if not fname.endswith(".json"): + continue + filepath = os.path.join(artifact_dir, fname) + with open(filepath, "r", encoding="utf-8") as f: + data = json.load(f) + result[data["name"]] = data + return result + + +def compare_artifacts(current, base): + diffs = {} + + all_names = sorted(set(list(current.keys()) + list(base.keys()))) + + for name in all_names: + if name not in current: + diffs[name] = "MISSING_IN_CURRENT" + elif name not in base: + diffs[name] = "MISSING_IN_BASE" + else: + cur = current[name] + base_artifact = base[name] + + if cur == base_artifact: + diffs[name] = "MATCH" + else: + diffs[name] = "DRIFT" + + return diffs, all_names + + +def format_diff(current, base_artifact, name): + if name not in current: + return ["=== {}: MISSING IN CURRENT ===".format(name)] + if name not in base_artifact: + return ["=== {}: MISSING IN BASE ===".format(name)] + + cur = current[name] + base_val = base_artifact[name] + lines = [] + + for key in sorted(set(list(cur.keys()) + list(base_val.keys()))): + cur_val = json.dumps(cur.get(key), indent=2, sort_keys=True, ensure_ascii=False) + base_val_str = json.dumps( + base_val.get(key), indent=2, sort_keys=True, ensure_ascii=False + ) + if cur_val != base_val_str: + diff_lines = list( + difflib.unified_diff( + base_val_str.splitlines(True), + cur_val.splitlines(True), + fromfile="base/{}".format(key), + tofile="current/{}".format(key), + lineterm="", + ) + ) + if diff_lines: + lines.append("--- Field: {} ---".format(key)) + lines.extend(diff_lines) + lines.append("") + + return lines + + +def write_summary_table(diffs, all_names, file=sys.stdout): + sep = "+" + "-" * 32 + "+" + "-" * 18 + "+" + file.write(sep + "\n") + file.write("| {:<30} | {:<16} |\n".format("Scenario", "Result")) + file.write(sep + "\n") + + drift_count = 0 + match_count = 0 + missing_count = 0 + + for name in all_names: + result = diffs[name] + if result == "MATCH": + match_count += 1 + elif result == "DRIFT": + drift_count += 1 + elif result.startswith("MISSING"): + missing_count += 1 + + file.write("| {:<30} | {:<16} |\n".format(name, result)) + + file.write(sep + "\n") + + total = len(all_names) + file.write( + "Summary: {} total, {} match, {} drift, {} missing\n".format( + total, match_count, drift_count, missing_count + ) + ) + + return drift_count > 0 + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="Compare characterization artifacts for drift detection" + ) + parser.add_argument("current_dir", help="Directory with current-run artifacts") + parser.add_argument("base_dir", help="Directory with base-branch artifacts") + parser.add_argument( + "--strict", + action="store_true", + help="Exit 1 if any drift is detected (default: exit 0 always)", + ) + args = parser.parse_args() + + current = load_artifacts(args.current_dir) + base = load_artifacts(args.base_dir) + + if not current: + print("ERROR: No artifacts found in current_dir: {}".format(args.current_dir)) + sys.exit(0 if not args.strict else 1) + + if not base: + print("WARNING: No artifacts found in base_dir: {}".format(args.base_dir)) + print("This is expected on the first run or after artifact expiry.") + print( + "Nothing to compare — this run's artifacts will serve as the new baseline." + ) + sys.exit(0) + + diffs, all_names = compare_artifacts(current, base) + + for name in all_names: + if diffs[name] != "MATCH": + diff_lines = format_diff(current, base, name) + if diff_lines: + for line in diff_lines: + print(line) + + has_drift = write_summary_table(diffs, all_names, file=sys.stdout) + + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if step_summary: + with open(step_summary, "a", encoding="utf-8") as f: + f.write("## CLI Characterization Drift Report\n\n") + f.write("
\n")
+            write_summary_table(diffs, all_names, file=f)
+            f.write("
\n") + f.write("\n") + if has_drift: + f.write( + "**Drift detected!** Review the diff above to confirm whether " + "the output change is intentional.\n" + ) + else: + f.write("No drift detected. All scenarios match the baseline.\n") + + if args.strict and has_drift: + sys.exit(1) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/download_base_artifact.py b/scripts/download_base_artifact.py new file mode 100644 index 0000000..45d4e11 --- /dev/null +++ b/scripts/download_base_artifact.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +""" +Download the most recent characterization artifact from the base branch. + +Uses the GitHub REST API via GITHUB_TOKEN. Designed for use in CI after the +current-run artifacts have been uploaded. + +Environment variables: + GITHUB_TOKEN — GitHub token for API calls + GITHUB_REPOSITORY — owner/repo (e.g. "postgresql-tools/migra") + GITHUB_HEAD_REF — PR head ref (used to find the base branch) + GITHUB_BASE_REF — PR base ref (e.g. "master") + +Output: + Downloads the artifact archive into tests/characterization/_base_artifacts/ + and extracts it there. + +Exit codes: + 0 — artifact downloaded and extracted successfully + 0 — no artifact found (expected on first run / after expiry); logs a warning + 1 — unexpected error +""" + +from __future__ import unicode_literals + +import json +import os +import sys +import tempfile +import zipfile +from urllib.request import Request, urlopen + +ARTIFACT_NAME_PREFIX = "characterization-" +OUTPUT_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "tests", + "characterization", + "_base_artifacts", +) + + +def api_get(url, token): + req = Request(url) + req.add_header("Authorization", "Bearer {}".format(token)) + req.add_header("Accept", "application/vnd.github.v3+json") + resp = urlopen(req) + return json.loads(resp.read().decode("utf-8")) + + +def api_get_stream(url, token): + req = Request(url) + req.add_header("Authorization", "Bearer {}".format(token)) + req.add_header("Accept", "application/vnd.github.v3+json") + return urlopen(req) + + +def main(): + token = os.environ.get("GITHUB_TOKEN") + repo = os.environ.get("GITHUB_REPOSITORY") + base_ref = os.environ.get("GITHUB_BASE_REF", "master") + + if not token: + print("WARNING: GITHUB_TOKEN not set, cannot download base artifact.") + sys.exit(0) + + if not repo: + print("WARNING: GITHUB_REPOSITORY not set, cannot download base artifact.") + sys.exit(0) + + api_base = "https://api.github.com/repos/{}".format(repo) + + # List artifacts for the repo, sorted by created_at desc + try: + data = api_get( + "{}/actions/artifacts?per_page=30".format(api_base), + token, + ) + except Exception as e: + print("WARNING: Failed to list artifacts: {}".format(e)) + sys.exit(0) + + artifacts = data.get("artifacts", []) + + # Filter: name starts with prefix, and was created on the base branch + # (GitHub artifacts store the workflow run's branch in the artifact metadata + # but we don't have direct branch info in the artifact list API response. + # Instead, we filter by artifacts whose workflow_run.head_branch matches base_ref.) + # Actually, the artifacts API response includes workflow_run.head_branch. + # Let's filter by that. + candidates = [] + for art in artifacts: + name = art.get("name", "") + if not name.startswith(ARTIFACT_NAME_PREFIX): + continue + wf_run = art.get("workflow_run", {}) + if not wf_run: + # Fall back to name matching — the old format might not have run info + candidates.append(art) + continue + head_branch = wf_run.get("head_branch", "") + if head_branch == base_ref: + candidates.append(art) + + if not candidates: + print( + "WARNING: No characterization artifact found for branch '{}'. " + "This is normal on the first run or after artifact expiry (5-day retention).".format( + base_ref + ) + ) + sys.exit(0) + + # Sort by created_at descending, pick the most recent + candidates.sort(key=lambda a: a.get("created_at", ""), reverse=True) + target = candidates[0] + artifact_id = target["id"] + created_at = target.get("created_at", "unknown") + + print( + "Downloading artifact id={} (created: {}) from branch '{}'".format( + artifact_id, created_at, base_ref + ) + ) + + # Download the artifact zip + download_url = "{}/actions/artifacts/{}/zip".format(api_base, artifact_id) + + os.makedirs(OUTPUT_DIR, exist_ok=True) + + try: + resp = api_get_stream(download_url, token) + data = resp.read() + except Exception as e: + print("WARNING: Failed to download artifact: {}".format(e)) + sys.exit(0) + + # Extract zip + with tempfile.TemporaryFile() as tmp: + tmp.write(data) + tmp.seek(0) + with zipfile.ZipFile(tmp) as zf: + zf.extractall(OUTPUT_DIR) + + # Verify something was extracted + extracted = os.listdir(OUTPUT_DIR) + print("Downloaded and extracted {} files to {}".format(len(extracted), OUTPUT_DIR)) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tests/characterization/README.md b/tests/characterization/README.md new file mode 100644 index 0000000..d415a46 --- /dev/null +++ b/tests/characterization/README.md @@ -0,0 +1,139 @@ +# CLI Characterization Tests + +## What this catches + +This suite **pins the current observable CLI output** of +`migra.command.run()` for every AI-powered flag combination, so that +future refactors of `command.py`, `ai_explain.py`, `ai_drift.py`, or +`db_inspector.py` get caught by a diff against a freshly generated +baseline. + +It detects **unintentional drift** in: +- Error messages (wording, formatting, exit codes) +- Output format (section headers, JSON structure, spacing) +- Flag interaction (combined `--explain --rollback --advise` output + shape) + +## What this does NOT catch + +- **Correctness** — there are no assertions like "the migration SQL + should contain ALTER TABLE". We only assert "the output changed + relative to the last CI run". +- **Live API behavior** — Anthropic is fully mocked. Real API changes + (new response formats, errors) will not be detected here. +- **Postgres behavior** — no real database assertions beyond fixture + loading (and even that is exercised only in `--from-file` scenarios). + +## Why nothing is committed to git + +Baseline files (`_artifacts/*.json`) are **generated in CI on every +run** and **compared within that same run**. They are never persisted +to the repository. + +This avoids: +- Stale baselines that drift from what the code actually produces +- PR noise from updating golden files on every intentional output + change +- Repository bloat on a small public project + +The trade-off is that comparison is against the **most recent CI run on +the base branch**, not against a curated golden file. If no prior +artifact exists (first run, or expired after 5-day retention), the +comparison step logs a warning and uploads the current run as the new +reference point — it does not fail the build. + +## Flag combinations covered + +See `tests/characterization/scenarios.py` for the full list. Currently +includes: + +| Category | Scenarios | +|----------|-----------| +| Error paths | Each AI flag (`--explain`, `--rollback`, `--advise`, `--generate`, `--explain-drift`) with missing API key | +| Error paths | Each AI flag with missing `anthropic` package (ImportError) | +| Error paths | `--explain-drift` with RuntimeError from mocked Anthropic | +| Empty diff | Each AI flag on identical schemas (via `"EMPTY"` sentinel) | +| Rich output | `--explain`, `--rollback`, `--advise` with `--from-file` and real fixtures | +| JSON output | `--explain --output json` with both empty and real diff | +| Combined flags | `--explain --rollback --advise` together | +| Generate | `--generate` with and without `--from-file` schema context | + +## How to read the CI job summary + +When the `characterize` CI job runs, it: + +1. **Phase A**: Runs every scenario, captures stdout/stderr/status, and + uploads the JSON artifacts with a 5-day retention. +2. **Phase B**: Downloads the most recent artifact from the base branch + (`master`), diffs scenario-by-scenario, and writes the result to the + job summary (visible in the GitHub Actions UI). + +The summary table looks like: + +``` ++--------------------------------+------------------+ +| Scenario | Result | ++--------------------------------+------------------+ +| explain_no_key | MATCH | +| explain_empty_diff | DRIFT | +| explain_with_file_everything | MATCH | +| ... | ... | ++--------------------------------+------------------+ +Summary: 21 total, 20 match, 1 drift, 0 missing +``` + +When **DRIFT** is reported, the unified diff for that scenario is +printed in the job logs. You can expand the "Compare against base +branch" step to see exactly which field changed. + +## Decision guide for maintainers + +### "Drift is fine, ship it" + +If the output change is intentional (e.g. you rewrote an error message, +added a new field to JSON output, reformatted a section header), simply +confirm the new output is correct and move on. **There is no baseline +file to update** — the next CI run on `master` will automatically +record the new output as the reference. + +### "Drift means a regression" + +If the diff reveals an unintended change (e.g. an error message lost +critical information, JSON output dropped a field, a combined flag +stopped printing a section), fix the regression in your branch. The +characterization suite is informational — it does not block merging — +but ignoring it means reviewers won't see a warning on the next PR +either. + +### "I want to see what the current output looks like" + +Each artifact JSON file is a structured capture of one scenario. You +can download the CI artifact zip from the Actions UI and inspect +individual `*.json` files. The format is: + +```json +{ + "name": "explain_empty_diff", + "args": ["--explain", "EMPTY", "EMPTY"], + "status": 0, + "stdout": "...", + "stderr": "...", + "description": "..." +} +``` + +## Upgrade path + +If this prove valuable and stable, the team may choose to promote it to +**committed baselines** — check the JSON artifacts into the repo under +`tests/characterization/baselines/` and compare against those instead +of against a CI artifact from the base branch. That would: + +- Make drift detection deterministic (same golden file every time) +- Remove the CI artifact download step +- Require intentional updates to baseline files (PR reviewers must + approve the diff) + +The current generate-in-CI approach is intentionally simpler to start +with, avoiding baseline staleness and PR noise until the team has lived +with the suite for a few weeks and trusts the signal. diff --git a/tests/characterization/__init__.py b/tests/characterization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/characterization/capture.py b/tests/characterization/capture.py new file mode 100644 index 0000000..18b99b9 --- /dev/null +++ b/tests/characterization/capture.py @@ -0,0 +1,126 @@ +from __future__ import unicode_literals + +import io +import json +import os +from unittest.mock import MagicMock, patch + +from .scenarios import SCENARIOS + +ARTIFACT_DIR = os.path.join(os.path.dirname(__file__), "_artifacts") + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _mock_anthropic(response_text): + mock_client = MagicMock() + mock_message = MagicMock() + mock_message.content = [MagicMock()] + mock_message.content[0].text = response_text + mock_client.messages.create.return_value = mock_message + return patch("anthropic.Anthropic", return_value=mock_client) + + +def _mock_import_error(): + import builtins + + original_import = builtins.__import__ + + def mock_import(name, *args, **kwargs): + if name == "anthropic": + raise ImportError("No module named 'anthropic'") + return original_import(name, *args, **kwargs) + + return patch("builtins.__import__", side_effect=mock_import) + + +def run_scenario(scenario): + from migra.command import parse_args, run + + out, err = outs() + + patches = [] + + if scenario.get("env") is not None: + patches.append(patch.dict("os.environ", scenario["env"], clear=True)) + + if "load_config_return" in scenario: + patches.append( + patch( + "migra.ai_explain.load_config", + return_value=scenario["load_config_return"], + ) + ) + + if scenario.get("mock_import_error"): + patches.append(_mock_import_error()) + + if scenario.get("mock_anthropic"): + response_text = scenario.get( + "mock_anthropic_response", + "Mock Anthropic response for characterization.", + ) + if scenario.get("mock_runtime_error"): + mock_client = MagicMock() + mock_client.messages.create.side_effect = RuntimeError("AI failure") + patches.append(patch("anthropic.Anthropic", return_value=mock_client)) + else: + patches.append(_mock_anthropic(response_text)) + + if scenario.get("mock_inspector"): + patches.append(patch("schemainspect.get_inspector", return_value=MagicMock())) + patches.append(patch("migra.db_inspector._fetch_table_sizes", return_value={})) + + for p in patches: + p.start() + + try: + args = parse_args(scenario["args"]) + status = run(args, out=out, err=err) + stdout = out.getvalue() + stderr = err.getvalue() + except Exception as exc: + stdout = out.getvalue() + stderr = err.getvalue() + "\nUNEXPECTED_EXCEPTION: {}".format(exc) + status = -1 + finally: + for p in reversed(patches): + p.stop() + + return { + "name": scenario["name"], + "args": scenario["args"], + "status": status, + "stdout": stdout, + "stderr": stderr, + "description": scenario.get("description", ""), + } + + +def capture_all(scenarios=None, artifact_dir=None): + if scenarios is None: + scenarios = SCENARIOS + if artifact_dir is None: + artifact_dir = ARTIFACT_DIR + + os.makedirs(artifact_dir, exist_ok=True) + + results = [] + for scenario in scenarios: + result = run_scenario(scenario) + results.append(result) + + filename = "{}.json".format(result["name"]) + filepath = os.path.join(artifact_dir, filename) + + json.dump( + result, + open(filepath, "w", encoding="utf-8"), + indent=2, + sort_keys=True, + ensure_ascii=False, + ) + + return results diff --git a/tests/characterization/scenarios.py b/tests/characterization/scenarios.py new file mode 100644 index 0000000..a5af51e --- /dev/null +++ b/tests/characterization/scenarios.py @@ -0,0 +1,347 @@ +from __future__ import unicode_literals + +SCENARIOS = [ + # ============================================================ + # ERROR-PATH SCENARIOS — every AI flag with no API key + # ============================================================ + { + "name": "explain_no_key", + "args": ["--explain", "EMPTY", "EMPTY"], + "env": {}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--explain with no API key -> error on stderr, exit 1", + }, + { + "name": "explain_missing_package", + "args": ["--explain", "EMPTY", "EMPTY"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": True, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--explain without anthropic package -> ImportError, exit 1", + }, + { + "name": "rollback_no_key", + "args": ["EMPTY", "EMPTY", "--rollback"], + "env": {}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--rollback with no API key -> error on stderr, exit 1", + }, + { + "name": "advise_no_key", + "args": ["--advise", "EMPTY", "EMPTY"], + "env": {}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--advise with no API key -> error on stderr, exit 1", + }, + { + "name": "generate_no_key", + "args": ["--generate", "add an email column to users", "EMPTY"], + "env": {}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--generate with no API key -> error on stderr, exit 1", + }, + { + "name": "explain_drift_no_key", + "args": [ + "--explain-drift", + "--from-db", + "postgresql://localhost/old", + "--to-db", + "postgresql://localhost/new", + ], + "env": {}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--explain-drift with no API key -> error on stderr, exit 1", + }, + { + "name": "explain_drift_missing_package", + "args": [ + "--explain-drift", + "--from-db", + "postgresql://localhost/old", + "--to-db", + "postgresql://localhost/new", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": False, + "mock_import_error": True, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "description": "--explain-drift without anthropic package -> ImportError, exit 1", + }, + { + "name": "explain_drift_runtime_error", + "args": [ + "--explain-drift", + "--from-db", + "postgresql://localhost/old", + "--to-db", + "postgresql://localhost/new", + "--api-key", + "sk-ant-test", + ], + "env": {}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": True, + "mock_inspector": True, + "needs_postgres": False, + "description": "--explain-drift with RuntimeError from Anthropic -> exit 1", + }, + # ============================================================ + # SUCCESS-PATH SCENARIOS — empty diff with mocked Anthropic + # ============================================================ + { + "name": "explain_empty_diff", + "args": ["--explain", "EMPTY", "EMPTY"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "Safe migration. Adds a column. Overall risk: LOW", + "description": "--explain on identical schemas -> 'No differences', exit 0", + }, + { + "name": "explain_json_empty_diff", + "args": ["--explain", "--output", "json", "EMPTY", "EMPTY"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "Safe migration. Adds a column. Overall risk: LOW", + "description": "--explain --output json on identical schemas -> JSON, exit 0", + }, + { + "name": "explain_rollback_advise_combined", + "args": ["--explain", "--advise", "EMPTY", "EMPTY", "--rollback"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "Safe migration. No destructive changes. Rollback: REVERSE.", + "description": "--explain --rollback --advise combined -> multiple sections, exit 0", + }, + { + "name": "rollback_empty_diff", + "args": ["EMPTY", "EMPTY", "--rollback"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "Rollback: CREATE TABLE public.users (id INT);", + "description": "--rollback on identical schemas -> 'No differences', exit 0", + }, + { + "name": "advise_empty_diff", + "args": ["--advise", "EMPTY", "EMPTY"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "Migration is LOW risk. No destructive operations detected.", + "description": "--advise on identical schemas -> 'No differences', exit 0", + }, + { + "name": "generate_success", + "args": ["--generate", "add an email column to users", "EMPTY"], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "ALTER TABLE public.users ADD COLUMN email TEXT;", + "description": "--generate with description -> generated SQL, exit 0", + }, + { + "name": "explain_drift_success", + "args": [ + "--explain-drift", + "--from-db", + "postgresql://localhost/old", + "--to-db", + "postgresql://localhost/new", + "--api-key", + "sk-ant-test", + ], + "env": {}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": True, + "needs_postgres": False, + "mock_anthropic_response": "Schema Drift Analysis: old -> new\nDetected: 2 new tables, 1 removed column.", + "description": "--explain-drift with mocked inspector -> drift analysis text, exit 0", + }, + # ============================================================ + # RICH-OUTPUT SCENARIOS — --from-file with real fixtures + Postgres + # ============================================================ + { + "name": "explain_with_file_everything", + "args": [ + "--explain", + "--unsafe", + "--from-file", + "tests/FIXTURES/everything/a.sql", + "tests/FIXTURES/everything/b.sql", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": True, + "mock_anthropic_response": "This migration adds several tables and columns. Overall risk: MEDIUM", + "description": "--explain --unsafe --from-file with everything fixture -> migration SQL + AI text", + }, + { + "name": "explain_json_with_file_everything", + "args": [ + "--explain", + "--output", + "json", + "--unsafe", + "--from-file", + "tests/FIXTURES/everything/a.sql", + "tests/FIXTURES/everything/b.sql", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": True, + "mock_anthropic_response": "This migration adds several tables and columns. Overall risk: MEDIUM", + "description": "--explain --output json with everything fixture -> JSON with explanation", + }, + { + "name": "rollback_with_file_everything", + "args": [ + "--rollback", + "--unsafe", + "--from-file", + "tests/FIXTURES/everything/a.sql", + "tests/FIXTURES/everything/b.sql", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": True, + "mock_anthropic_response": "DROP TABLE public.new_table; ALTER TABLE public.users DROP COLUMN email;", + "description": "--rollback --unsafe with everything fixture -> migration SQL + rollback text", + }, + { + "name": "advise_with_file_everything", + "args": [ + "--advise", + "--unsafe", + "--from-file", + "tests/FIXTURES/everything/a.sql", + "tests/FIXTURES/everything/b.sql", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": True, + "mock_anthropic_response": "Migration risk assessment: LOW. All changes are additive.", + "description": "--advise --unsafe with everything fixture -> migration SQL + advisory text", + }, + { + "name": "explain_rollback_advise_with_file_enumdeps", + "args": [ + "--explain", + "--rollback", + "--advise", + "--unsafe", + "--from-file", + "tests/FIXTURES/enumdeps/a.sql", + "tests/FIXTURES/enumdeps/b.sql", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": True, + "mock_anthropic_response": "Combined: explain + rollback + advise for enum dependency changes.", + "description": "--explain --rollback --advise with enumdeps fixture -> multi-section", + }, + { + "name": "generate_with_file_everything", + "args": [ + "--generate", + "add a created_at timestamp column to all tables", + "--unsafe", + "--from-file", + "tests/FIXTURES/everything/a.sql", + "tests/FIXTURES/everything/b.sql", + ], + "env": {"ANTHROPIC_API_KEY": "sk-ant-test"}, + "load_config_return": None, + "mock_anthropic": True, + "mock_import_error": False, + "mock_runtime_error": False, + "mock_inspector": False, + "needs_postgres": False, + "mock_anthropic_response": "ALTER TABLE public.users ADD COLUMN created_at TIMESTAMPTZ DEFAULT NOW();", + "description": "--generate with --from-file and schema context -> generated SQL", + }, +] diff --git a/tests/characterization/test_capture.py b/tests/characterization/test_capture.py new file mode 100644 index 0000000..a806349 --- /dev/null +++ b/tests/characterization/test_capture.py @@ -0,0 +1,151 @@ +from __future__ import unicode_literals + +import json +import os +import shutil + +from .capture import ARTIFACT_DIR, SCENARIOS, capture_all, run_scenario + + +class TestCharacterizationCapture: + def test_all_scenarios_captured(self): + tmpdir = ARTIFACT_DIR + "_tmp" + os.makedirs(tmpdir, exist_ok=True) + try: + results = capture_all(artifact_dir=tmpdir) + + for result in results: + name = result["name"] + assert isinstance( + result["status"], int + ), "{}: status should be int, got {}".format( + name, type(result["status"]) + ) + assert isinstance( + result["stdout"], str + ), "{}: stdout should be str".format(name) + assert isinstance( + result["stderr"], str + ), "{}: stderr should be str".format(name) + + artifact_files = [f for f in os.listdir(tmpdir) if f.endswith(".json")] + assert len(artifact_files) == len( + results + ), "Expected {} artifact files, found {}".format( + len(results), len(artifact_files) + ) + + for fname in artifact_files: + filepath = os.path.join(tmpdir, fname) + with open(filepath, "r", encoding="utf-8") as f: + data = json.load(f) + assert "name" in data + assert "status" in data + assert "stdout" in data + assert "stderr" in data + assert "args" in data + + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_no_key_scenarios_exit_one(self): + no_key_scenarios = [ + s + for s in SCENARIOS + if "no_key" in s["name"] and not s.get("needs_postgres") + ] + + for scenario in no_key_scenarios: + result = run_scenario(scenario) + assert ( + result["status"] == 1 + ), "{} expected exit 1, got {}: stderr={!r}".format( + scenario["name"], result["status"], result["stderr"][:200] + ) + assert result["stderr"], "{} expected stderr output, got empty".format( + scenario["name"] + ) + + def test_missing_package_scenarios_exit_one(self): + missing_pkg_scenarios = [ + s + for s in SCENARIOS + if s.get("mock_import_error") and not s.get("needs_postgres") + ] + + for scenario in missing_pkg_scenarios: + result = run_scenario(scenario) + assert ( + result["status"] == 1 + ), "{} expected exit 1, got {}: stderr={!r}".format( + scenario["name"], result["status"], result["stderr"][:200] + ) + assert ( + "AI extras" in result["stderr"] or "requires the AI" in result["stderr"] + ), "{} expected 'AI extras' error, got: {!r}".format( + scenario["name"], result["stderr"][:200] + ) + + def test_success_empty_diff_scenarios_exit_zero(self): + success_scenarios = [ + s + for s in SCENARIOS + if "empty_diff" in s["name"] + and s.get("mock_anthropic") + and not s.get("needs_postgres") + ] + + for scenario in success_scenarios: + result = run_scenario(scenario) + assert ( + result["status"] == 0 + ), "{} expected exit 0, got {}: stderr={!r}".format( + scenario["name"], result["status"], result["stderr"][:200] + ) + stdout = result["stdout"] + if "--output json" in " ".join(scenario["args"]) or "--output" in " ".join( + scenario["args"] + ): + import json + + data = json.loads(stdout) + assert "version" in data + assert "summary" in data + assert data["summary"]["total_statements"] == 0 + else: + assert ( + "No schema differences detected" in stdout + or "The schemas are identical" in stdout + ), "{} expected 'No differences' in stdout, got: {!r}".format( + scenario["name"], stdout[:300] + ) + + def test_explain_drift_runtime_error_exits_one(self): + scenario = next( + s for s in SCENARIOS if s["name"] == "explain_drift_runtime_error" + ) + result = run_scenario(scenario) + assert result["status"] == 1, "Expected exit 1, got {}: stderr={!r}".format( + result["status"], result["stderr"][:200] + ) + + def test_generate_success_has_output(self): + scenario = next(s for s in SCENARIOS if s["name"] == "generate_success") + result = run_scenario(scenario) + assert result["status"] == 0, "Expected exit 0, got {}: stderr={!r}".format( + result["status"], result["stderr"][:200] + ) + assert result["stdout"], "Expected stdout output" + assert ( + "ALTER TABLE" in result["stdout"] or result["stdout"].strip() + ), "Expected SQL-like output, got: {!r}".format(result["stdout"][:200]) + + def test_explain_drift_success_has_drift_analysis(self): + scenario = next(s for s in SCENARIOS if s["name"] == "explain_drift_success") + result = run_scenario(scenario) + assert result["status"] == 0, "Expected exit 0, got {}: stderr={!r}".format( + result["status"], result["stderr"][:200] + ) + assert ( + "Drift Analysis" in result["stdout"] or "Drift" in result["stdout"] + ), "Expected drift analysis in stdout, got: {!r}".format(result["stdout"][:300])