diff --git a/.env.example b/.env.example index 35464d8..f508bff 100644 --- a/.env.example +++ b/.env.example @@ -70,10 +70,12 @@ WATERMARKS_SERVER_API_KEY= # WATERMARKS_SERVER_VERSION= # ─── Vendor text-watermark detection ──────────────────────────── -# Optional Google Gemini API key. When set, the service can run Google's -# official SynthID-text watermark detector via POST /detect and the -# detect_before / detect_after clean options. Env only — never on argv. -# Privacy: text is sent to Google only when this key is configured. +# Optional Google Gemini API key. Env only — never on argv. The Gemini +# SynthID-text detector is currently DISABLED: the generateContent API +# exposes no DETECT_TEXT_WATERMARK task type, so /capabilities always +# reports text_detectors.gemini-synthid-text as false and setting this key +# enables nothing today. Kept so the wiring is ready if the task type ships; +# text would then be sent to Google only when this key is configured. # WATERMARKS_GEMINI_API_KEY= # WATERMARKS_GEMINI_MODEL=gemini-2.5-flash # WATERMARKS_GEMINI_TIMEOUT=30 diff --git a/README.md b/README.md index 3e64c05..a2da6fc 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@

PyPI - Downloads - Python 3.10+ - CI Release - Stars - License: MIT - Visitors + Python 3.10+ + Downloads + CI + Stars + License: MIT + Visitors

Tools for finding and removing AI provenance signals from files you own. Four channels are covered: hidden Unicode in text, statistical token watermarks, visible marks burned into images, and metadata such as C2PA, EXIF, and XMP. @@ -296,7 +296,8 @@ Best-effort pixel-domain removal is opt-in on image cleans: `--remove-synthid` ( Optional, stdlib-first, fail-soft: - `score_stylometry.py` — zero-LLM stylometry (burstiness, MATTR, AI-phrase density) with confidence bands and `--explain`. -- `text_detectors.py` / `detect_text_watermark.py` — Gemini's official SynthID-text detector (needs `WATERMARKS_GEMINI_API_KEY`; env only) and a MarkLLM research harness (same-scheme-config only, not a vendor oracle). Claude detection is reserved but unavailable until a public API ships. +- `text_detectors.py` / `detect_text_watermark.py` — a MarkLLM research harness (same-scheme-config only, not a vendor oracle). Both vendor detectors are reserved but unavailable, and `/capabilities` always reports them `false`: Gemini's official SynthID-text detector is wired for `WATERMARKS_GEMINI_API_KEY` (env only) but the Gemini API exposes no `DETECT_TEXT_WATERMARK` task type, and Claude detection waits on a public API. +- MarkLLM detection reports a document-level `verdict` (`DETECTED`, `NOT_DETECTED`, `INCONCLUSIVE`, `UNSUPPORTED`, `ERROR`) alongside the raw `detector_verdict`. Only a below-threshold score with a confirmed key/provenance match and enough scored tokens reads as `NOT_DETECTED`; unknown provenance, a near-threshold score, or a short sample produces `INCONCLUSIVE`. An unsupported scheme produces `UNSUPPORTED`, and a detector result that cannot be scored produces `ERROR`. Provenance comes from the `.wm.json` sidecar the `watermark` subcommand writes, or from an operator `--key-id` assertion. - `inspect_text.py --stylometry` and `rewrite_text.py` MarkLLM before/after hooks. ```bash diff --git a/compose.yaml b/compose.yaml index 0092d5e..bb40cee 100644 --- a/compose.yaml +++ b/compose.yaml @@ -27,8 +27,8 @@ services: environment: # Empty by default (no auth). Set to require `Authorization: Bearer `. WATERMARKS_SERVER_API_KEY: ${WATERMARKS_SERVER_API_KEY:-} - # Optional Gemini API key enabling vendor SynthID-text watermark - # detection via /detect and detect_before/after (see .env.example). + # Optional Gemini API key. The vendor SynthID-text detector is + # currently disabled and this key enables nothing (see .env.example). WATERMARKS_GEMINI_API_KEY: ${WATERMARKS_GEMINI_API_KEY:-} WATERMARKS_GEMINI_MODEL: ${WATERMARKS_GEMINI_MODEL:-} # Optional SynthID image scorer sidecar (heavy profile). diff --git a/skills/remove-ai-marks/references/service-mode.md b/skills/remove-ai-marks/references/service-mode.md index a945908..1246557 100644 --- a/skills/remove-ai-marks/references/service-mode.md +++ b/skills/remove-ai-marks/references/service-mode.md @@ -52,7 +52,7 @@ field and writes it to the output path itself. | GET | `/health` | — | `{"ok": true, "version": ...}` | | GET | `/capabilities` | — | optional tools / backends present | | GET | `/openapi.json` | — | dynamically generated OpenAPI 3.0.3 spec | -| POST | `/inspect` | `{"file": "", "name": "notes.md"}` | `{"ok", "kind", "suspicious", "report"}` | +| POST | `/inspect` | `{"file": "", "name": "notes.md", "detect": true}` | `{"ok", "kind", "suspicious", "detection_status", "report"}` | | POST | `/detect` | `{"file": "", "name": "notes.txt"}` | `{"ok", "kind", "detections": [...]}` | | POST | `/clean` | `{"file": "", "name": "notes.md", "options": {...}}` | `{"ok", "kind", "cleaned": "", "report"}` | @@ -61,6 +61,15 @@ unrecognized formats answer `kind: "unknown"` (`/inspect`) or 400 (`/clean`). When writing a temp file for pasted text, keep a known extension (`.txt` / `.md`) in the `name` you send. +For `/inspect`, `detection_status` is `DETECTED`, `INCONCLUSIVE`, +`NOT_DETECTED`, or `NOT_RUN`. `INCONCLUSIVE` means an available detector ran but +could not rule out a watermark; it sets `suspicious: true` conservatively but is +not a confirmed detection. A configured detector that ran and failed (timeout, +crash, unreadable output) also aggregates to `INCONCLUSIVE`, never `NOT_RUN`. +`NOT_RUN` means detection was not requested or no detector was configured. +Pass `"detect": true` only with consent when a configured vendor detector may +send text to its provider. + The machine-readable contract lives at `$WM/openapi.json` — plug it into any OpenAPI tooling instead of hand-rolling clients. @@ -92,15 +101,18 @@ default unless the user asked in-place) and summarize `report` honestly. ## Watermark detection before/after (when configured) -When `/capabilities` reports a vendor detector (`text_detectors.gemini-synthid-text`) -or an image scorer (`scorers.synthid_http` / `scorers.synthid`), measure the +When `/capabilities` reports a text detector (`text_detectors.markllm`) or an +image scorer (`scorers.synthid_http` / `scorers.synthid`), measure the result by detecting before and after cleaning — either `POST /detect` or fold it into the clean with `{"options": {"detect_before": true, "detect_after": true}}` (returns `text_detectors.before/after` for text or `synthid_before/after` for -images). Vendor detection sends text to the configured provider (Gemini) — -only use it with user consent, and report the vendor's verdict honestly -(Gemini = Google's official SynthID-text detector; MarkLLM is same-config-only -research; Claude's detector is not public yet). +images). MarkLLM is same-config-only research, not a vendor oracle; report its +verdict honestly. + +No vendor text detector is reachable today: `text_detectors.gemini-synthid-text` +and `text_detectors.claude-text` always report `false` in `/capabilities` (see +the detector list in the README). If one does become available, it sends text +to that provider — only use it with explicit user consent. ## Aggregate audits (directories / websites) diff --git a/skills/remove-ai-marks/scripts/detect_text_watermark.py b/skills/remove-ai-marks/scripts/detect_text_watermark.py index 36b9c11..62b6d5a 100755 --- a/skills/remove-ai-marks/scripts/detect_text_watermark.py +++ b/skills/remove-ai-marks/scripts/detect_text_watermark.py @@ -9,6 +9,23 @@ valid against the SAME scheme config + keys used at generation. It cannot certify that a vendor detector will fail on the given text. +Decision model: + A detector may say what it observed; only the decision engine may say what + that observation means. Each report therefore carries: + - detector_verdict: the raw MarkLLM observation (DETECTED / NOT_DETECTED) + - verdict: the document-level decision, one of + DETECTED score at/above threshold + NOT_DETECTED below threshold AND provenance/key match confirmed AND + enough scored tokens + INCONCLUSIVE below threshold but provenance unknown/mismatched, or + score inside the abstention band, or too few (or an + unknown number of) scored tokens + UNSUPPORTED document provenance declares a scheme we cannot run + ERROR detector returned no score/threshold + A negative with unknown provenance must never be read as "watermark-free". + Provenance comes from a sidecar (.wm.json) written by the `watermark` + subcommand, or from an operator assertion via --key-id. + Subcommands: detect run detection on a text file with a known scheme/config watermark generate watermarked (and optionally unwatermarked) sample text @@ -24,9 +41,11 @@ from __future__ import annotations import argparse +import hashlib import json import os import sys +from datetime import datetime, timezone from pathlib import Path SCRIPTS_DIR = Path(__file__).resolve().parent @@ -37,6 +56,7 @@ emit_json, eprint, read_text_input, + which, ) # Scheme name as the user types it -> MarkLLM algorithm name (config/{ALG}.json). @@ -48,6 +68,31 @@ DEFAULT_MODEL = "facebook/opt-1.3b" +# Document-level verdicts produced by the decision engine. A detector may say +# what it observed; only the decision engine may say what that observation +# means. The engine never lets a negative with unknown provenance, a missing +# score, an unsupported scheme, or a too-short sample collapse into "clean". +VERDICT_DETECTED = "DETECTED" +VERDICT_NOT_DETECTED = "NOT_DETECTED" +VERDICT_INCONCLUSIVE = "INCONCLUSIVE" +VERDICT_UNSUPPORTED = "UNSUPPORTED" +VERDICT_ERROR = "ERROR" + +# Relative abstention band around the config threshold. Scores within +# |score - threshold| / threshold < band are treated as insufficient evidence +# (SynthID's published design abstains to preserve target error rates; KGW's +# z-test is likewise unreliable in the near-threshold region). +DEFAULT_ABSTENTION_BAND = 0.10 + +# Below this many tokens, a negative is not a strong negative even with +# matching provenance: KGW/SynthID detect via low-stakes word choices, which +# are too few in short text (measured: 677 chars -> miss, 1912 chars -> hit). +DEFAULT_MIN_TOKENS = 200 + +# Provenance sidecar suffix: .wm.json records scheme/key/config_hash +# for text generated under a known key, making detection deterministic. +SIDECAR_SUFFIX = ".wm.json" + # Algorithm configs are ~200 B (KGW/SynthID). Cap well above that so a crafted # or accidental huge file is refused before either this script or upstream # reads it into memory. @@ -87,7 +132,11 @@ def resolve_device(raw: str | None) -> str: def _load_algorithm( upstream: Path, alg: str, config: Path, model: str, device: str, offline: bool = False ): - """Import the checkout and build an ``AutoWatermark`` instance.""" + """Import the checkout and build an ``AutoWatermark`` instance. + + Returns ``(watermark, tokenizer)``; the tokenizer is needed for the + decision engine's token-length calibration. + """ sys.path.insert(0, str(upstream)) try: from transformers import AutoModelForCausalLM, AutoTokenizer @@ -117,11 +166,12 @@ def _load_algorithm( do_sample=True, no_repeat_ngram_size=4, ) - return AutoWatermark.load( + wm = AutoWatermark.load( alg, algorithm_config=str(config), transformers_config=transformers_config, ) + return wm, tokenizer def _threshold_from_config(config: Path) -> float | None: @@ -136,6 +186,135 @@ def _threshold_from_config(config: Path) -> float | None: return None +def _sha256_file(path: Path) -> str: + """Stable content hash of a config file, for provenance matching.""" + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def _git_commit(upstream: Path) -> str | None: + """Best-effort pinned-commit id of the MarkLLM checkout (contract version).""" + git = which("git") + if git is None: + return None + try: + import subprocess + + out = subprocess.run( + [git, "-C", str(upstream), "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + commit = out.stdout.strip() + return commit or None + + +def _sidecar_path_for(input_path: str) -> Path | None: + """Return .wm.json if a provenance sidecar exists next to the input.""" + if input_path == "-": + return None + sidecar = Path(input_path + SIDECAR_SUFFIX) + if not sidecar.is_file(): + return None + try: + if sidecar.stat().st_size > MAX_CONFIG_BYTES: + return None + except OSError: + return None + return sidecar + + +def _read_sidecar(path: Path) -> dict: + """Read a provenance sidecar; malformed sidecars degrade to {} (unknown).""" + try: + data = json.loads(path.read_text("utf-8")) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def _count_tokens(tokenizer: object, text: str) -> int | None: + """Count input tokens via the loaded tokenizer; None if unavailable.""" + encode = getattr(tokenizer, "encode", None) + if encode is None: + return None + try: + ids = encode(text) + if isinstance(ids, int): + return ids + return len(ids) + except Exception: + return None + + +def _decide( + *, + detector_verdict: str, + score: float | None, + threshold: float | None, + provenance_match: bool | None, + scored_tokens: int | None, + min_tokens: int, + abstention_band: float, +) -> tuple[str, str]: + """Map detector observations + provenance to a document-level verdict. + + The detector says what it observed (``detector_verdict``); this is the + decision engine that decides what the observation means. The invariant: + a negative must never become "clean" when the key/provenance is unknown, + the score sits in the abstention band, or the sample is too short. + """ + if detector_verdict == VERDICT_DETECTED: + # Aggressive posture: a positive detector observation is always + # reported as DETECTED, even when its score sits inside the + # abstention band or the config exposes no threshold. The band + # and min-length checks only qualify negatives. + if score is not None and threshold is not None: + return ( + VERDICT_DETECTED, + f"score {score:.4f} at/above threshold {threshold:.4f}", + ) + return (VERDICT_DETECTED, "detector reported watermarked") + if score is None or threshold is None: + return VERDICT_ERROR, "detector returned no score or threshold" + if abstention_band > 0 and threshold != 0: + rel = abs(score - threshold) / abs(threshold) + if rel < abstention_band: + return ( + VERDICT_INCONCLUSIVE, + f"score {score:.4f} within {abstention_band:.0%} abstention band " + f"of threshold {threshold:.4f}", + ) + if provenance_match is not True: + return ( + VERDICT_INCONCLUSIVE, + "detector below threshold but provenance/key match not confirmed", + ) + if min_tokens > 0 and scored_tokens is None: + return ( + VERDICT_INCONCLUSIVE, + f"scored token count unknown (minimum {min_tokens} tokens)", + ) + if min_tokens > 0 and scored_tokens < min_tokens: + return ( + VERDICT_INCONCLUSIVE, + f"below minimum calibrated length ({scored_tokens} < {min_tokens} tokens)", + ) + return ( + VERDICT_NOT_DETECTED, + f"score {score:.4f} below threshold {threshold:.4f} with confirmed provenance/key match", + ) + + def _resolve_config(upstream: Path, alg: str, config: str | None) -> Path: path = Path(config).expanduser().resolve() if config else upstream / "config" / f"{alg}.json" if not path.is_file(): @@ -154,13 +333,65 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path, alg: str) -> int: eprint(f"not a file: {args.path}") return 2 text = read_text_input(args.path, allow_binary=args.force_text) + sidecar_path = _sidecar_path_for(args.path) device = resolve_device(args.device) try: config = _resolve_config(upstream, alg, args.config) threshold = _threshold_from_config(config) - wm = _load_algorithm(upstream, alg, config, args.model, device, offline=args.offline) + except _Unavailable as e: + eprint(str(e)) + return 3 + + # --- Provenance resolution (decision engine input) --- + # A sidecar (.wm.json) records how text generated under a known key + # was marked. Without it, provenance is UNKNOWN unless the operator asserts + # a key identity with --key-id. Resolved before the (expensive) model load + # so UNSUPPORTED provenance fails fast. + sidecar = _read_sidecar(sidecar_path) if sidecar_path is not None else {} + sidecar_scheme = sidecar.get("scheme") + sidecar_scheme_l = str(sidecar_scheme).lower() if sidecar_scheme else None + sidecar_config_hash = sidecar.get("config_hash") + try: + config_hash = _sha256_file(config) + except OSError as e: + eprint(f"cannot read watermarking config: {e}") + return 3 + + # UNSUPPORTED: the document's provenance declares a scheme we cannot run. + if sidecar_scheme is not None and sidecar_scheme_l not in SCHEMES: + payload = { + "available": True, + "upstream_dir": str(upstream), + "scheme": alg, + "config": str(config), + "model": args.model, + "device": device, + "verdict": VERDICT_UNSUPPORTED, + "verdict_reason": f"document provenance declares unsupported scheme '{sidecar_scheme}'", + "detector_verdict": None, + "is_watermarked": None, + "score": None, + "threshold": threshold, + "provenance_match": False, + "provenance": sidecar, + "config_hash": config_hash, + "implementation_commit": args._implementation_commit, + "input_chars": len(text), + "input_tokens": None, + "effective_scored_tokens": None, + } + if args.json: + emit_json(payload) + else: + print(f"{alg}: {VERDICT_UNSUPPORTED} (unsupported scheme '{sidecar_scheme}')") + return 0 + + try: + wm, tokenizer = _load_algorithm( + upstream, alg, config, args.model, device, offline=args.offline + ) result = wm.detect_watermark(text, return_dict=True) except _Unavailable as e: eprint(str(e)) @@ -176,6 +407,48 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path, alg: str) -> int: except (TypeError, ValueError): score = None + key_id = args.key_id or sidecar.get("key_id") + # provenance_match: True only when the sidecar's config matches the config + # we detected with (content hash), or the *operator* asserted a key id on + # the command line. A sidecar key_id alone is document-supplied and cannot + # confirm provenance: the `watermark` subcommand always writes config_hash, + # so a sidecar without one did not come from this tool. + if sidecar_config_hash is not None: + provenance_match = sidecar_config_hash == config_hash + elif args.key_id is not None: + provenance_match = True + else: + provenance_match = None + # A sidecar declaring a different *supported* scheme is still a mismatch + # for the requested detector. Compare normalized algorithm names so the + # synthid/synthid-text aliases match each other. + if ( + sidecar_scheme is not None + and SCHEMES.get(sidecar_scheme_l) is not None + and SCHEMES.get(sidecar_scheme_l) != alg + ): + provenance_match = False + + detector_verdict = VERDICT_DETECTED if is_watermarked else VERDICT_NOT_DETECTED + input_tokens = _count_tokens(tokenizer, text) + effective = result.get("num_tokens") + try: + effective = int(effective) if effective is not None else None + except (TypeError, ValueError): + effective = None + if effective is None: + effective = input_tokens + + verdict, reason = _decide( + detector_verdict=detector_verdict, + score=score, + threshold=threshold, + provenance_match=provenance_match, + scored_tokens=effective, + min_tokens=args.min_tokens, + abstention_band=args.abstention_band, + ) + payload = { "available": True, "upstream_dir": str(upstream), @@ -183,22 +456,72 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path, alg: str) -> int: "config": str(config), "model": args.model, "device": device, + "verdict": verdict, + "verdict_reason": reason, + "detector_verdict": detector_verdict, "is_watermarked": is_watermarked, "score": score, "threshold": threshold, + "provenance_match": provenance_match, + "key_id": key_id, + "provenance": sidecar or None, + "config_hash": config_hash, + "implementation_commit": args._implementation_commit, + "input_chars": len(text), + "input_tokens": input_tokens, + "effective_scored_tokens": effective, + "min_tokens": args.min_tokens, + "abstention_band": args.abstention_band, } if args.json: emit_json(payload) else: - label = "watermarked" if is_watermarked else "not watermarked" + label = { + VERDICT_DETECTED: "detected", + VERDICT_NOT_DETECTED: "not detected", + VERDICT_INCONCLUSIVE: "inconclusive", + VERDICT_ERROR: "error", + VERDICT_UNSUPPORTED: "unsupported", + }.get(verdict, verdict) score_txt = f"{score:.4f}" if score is not None else "n/a" thresh_txt = f"{threshold:.4f}" if threshold is not None else "n/a" - print(f"{alg}: {label} (score {score_txt}, threshold {thresh_txt})") + pm = ( + "matched" + if provenance_match is True + else "mismatched" + if provenance_match is False + else "unknown" + ) + print( + f"{alg}: {label} (score {score_txt}, threshold {thresh_txt}, " + f"provenance/key match: {pm})" + ) + if verdict == VERDICT_INCONCLUSIVE: + print(f" reason: {reason}") + if verdict != VERDICT_DETECTED: + print(" This result does NOT establish that the document is watermark-free.") return 0 +_SECRET_KEY_PARTS = ("key", "seed", "secret", "salt", "password") + + +def _redact_secrets(value: object) -> object: + """Strip secret-bearing fields from a config before it travels with the sample.""" + if isinstance(value, dict): + return { + k: "[redacted]" + if any(part in str(k).lower() for part in _SECRET_KEY_PARTS) + else _redact_secrets(v) + for k, v in value.items() + } + if isinstance(value, list): + return [_redact_secrets(v) for v in value] + return value + + def _cmd_watermark(args: argparse.Namespace, upstream: Path, alg: str) -> int: prompt = read_text_input(args.prompt, allow_binary=args.force_text) @@ -206,7 +529,19 @@ def _cmd_watermark(args: argparse.Namespace, upstream: Path, alg: str) -> int: try: config = _resolve_config(upstream, alg, args.config) - wm = _load_algorithm(upstream, alg, config, args.model, device, offline=args.offline) + except _Unavailable as e: + eprint(str(e)) + return 3 + try: + config_hash = _sha256_file(config) + except OSError as e: + eprint(f"cannot read watermarking config: {e}") + return 3 + + try: + wm, _tokenizer = _load_algorithm( + upstream, alg, config, args.model, device, offline=args.offline + ) if args.seed is not None: import torch @@ -233,6 +568,31 @@ def _cmd_watermark(args: argparse.Namespace, upstream: Path, alg: str) -> int: (sys.stderr if args.json else sys.stdout).write(watermarked) else: atomic_write_text(Path(wm_out), watermarked) + # Provenance sidecar: record how this sample was marked so detection + # of it later is deterministic instead of guesswork. key_id is an + # identifier, never the raw secret. The config is embedded for human + # inspection with secret-bearing fields redacted; a later detection + # verifies provenance through config_hash alone, never these values. + try: + params = _redact_secrets(json.loads(config.read_text("utf-8"))) + except (OSError, ValueError): + params = None + sidecar = { + "scheme": alg, + "key_id": args.key_id, + "tokenizer": args.model, + "watermark_parameters": params, + "config_hash": config_hash, + "generator": args.model, + "generation_settings": { + "max_new_tokens": args.max_new_tokens, + "min_length": args.min_length, + "seed": args.seed, + }, + "implementation_commit": args._implementation_commit, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + atomic_write_text(Path(wm_out + SIDECAR_SUFFIX), json.dumps(sidecar, indent=2)) if unwatermarked is not None: atomic_write_text(Path(args.unwatermarked_output), unwatermarked) @@ -306,6 +666,13 @@ def _add_common(p: argparse.ArgumentParser) -> None: action="store_true", help="Process input even when it looks like a binary container", ) + p.add_argument( + "--key-id", + default=None, + help="Identifier of the watermark key this text was generated under " + "(never the raw secret). Establishes provenance_match for detection; " + "recorded in the sidecar by `watermark`.", + ) def _apply_rlimit_as(bytes_: int | None) -> None: @@ -345,6 +712,18 @@ def main() -> int: detect = sub.add_parser("detect", help="Detect a scheme watermark in text") detect.add_argument("path", help="Text file to detect on, or - for stdin") _add_common(detect) + detect.add_argument( + "--min-tokens", + type=int, + default=DEFAULT_MIN_TOKENS, + help=f"Minimum scored tokens for a strong negative (default: {DEFAULT_MIN_TOKENS})", + ) + detect.add_argument( + "--abstention-band", + type=float, + default=DEFAULT_ABSTENTION_BAND, + help=f"Relative near-threshold abstention band (default: {DEFAULT_ABSTENTION_BAND})", + ) detect.add_argument("--json", action="store_true", help="Emit JSON on stdout") detect.set_defaults(handler=_cmd_detect) @@ -385,6 +764,9 @@ def main() -> int: return 3 alg = SCHEMES[args.scheme] + # Contract version: the checkout commit this run detects against. Used in + # reports so results stay comparable across upgrades. + args._implementation_commit = _git_commit(upstream) return args.handler(args, upstream, alg) diff --git a/skills/remove-ai-marks/scripts/layer_b_http.py b/skills/remove-ai-marks/scripts/layer_b_http.py index 01141c1..c57a46c 100644 --- a/skills/remove-ai-marks/scripts/layer_b_http.py +++ b/skills/remove-ai-marks/scripts/layer_b_http.py @@ -202,7 +202,6 @@ def _reject_json_constant(_value: str) -> Any: def _read_json_object(response: Any, limit: int) -> dict[str, Any]: - content_lengths = _header_values(response.headers, "Content-Length") if content_lengths: normalized = {value.strip() for value in content_lengths} diff --git a/skills/remove-ai-marks/scripts/server.py b/skills/remove-ai-marks/scripts/server.py index d72815c..a79d340 100755 --- a/skills/remove-ai-marks/scripts/server.py +++ b/skills/remove-ai-marks/scripts/server.py @@ -59,6 +59,9 @@ # stays well under MAX_INPUT_BYTES for the same cap. MAX_BODY_BYTES = MAX_INPUT_BYTES + (MAX_INPUT_BYTES >> 1) +TEXT_DETECTION_STATUSES = ("DETECTED", "INCONCLUSIVE", "NOT_DETECTED", "NOT_RUN") +_UNRESOLVED_TEXT_VERDICTS = frozenset({"INCONCLUSIVE", "UNSUPPORTED", "ERROR"}) + ALLOWED_CLEAN_OPTIONS = { "nfkc": bool, "aggressive_homoglyphs": bool, @@ -78,6 +81,38 @@ def _json_ok(payload: dict[str, Any]) -> bytes: return json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") +def _text_detection_status(reports: list[dict[str, Any]]) -> str: + """Aggregate available text-detector reports without treating unknown as clean.""" + unresolved = False + not_detected = False + for report in reports: + if not isinstance(report, dict): + continue + if report.get("available") is not True: + # A detector that was configured to run but reported unavailable + # ran and failed (timeout, crash, bad output). That is unresolved + # evidence, not "detection was never requested". + if report.get("configured") is True: + unresolved = True + continue + verdict = report.get("verdict") + if report.get("is_watermarked") is True or verdict == "DETECTED": + return "DETECTED" + if verdict in _UNRESOLVED_TEXT_VERDICTS: + unresolved = True + elif verdict == "NOT_DETECTED" or report.get("is_watermarked") is False: + not_detected = True + else: + # An available detector that produced no recognized decision did + # run, but cannot support a clean result. + unresolved = True + if unresolved: + return "INCONCLUSIVE" + if not_detected: + return "NOT_DETECTED" + return "NOT_RUN" + + def capabilities() -> dict[str, Any]: return { "version": VERSION, @@ -245,6 +280,16 @@ def _clean_request_schema() -> dict[str, Any]: type="string", enum=["text", "image", "container", "unknown"] ), "suspicious": _schema(type="boolean"), + "detection_status": _schema( + type="string", + enum=list(TEXT_DETECTION_STATUSES), + description=( + "Aggregate text-watermark detector state. INCONCLUSIVE means " + "the service cannot rule out a watermark; it is not a confirmed " + "detection. A configured detector that ran and failed reports " + "INCONCLUSIVE, not NOT_RUN." + ), + ), "report": _schema(type="object"), }, ) @@ -498,6 +543,7 @@ def _handle_inspect(self, data: bytes, name: str, body: dict[str, Any]) -> None: "note": "unrecognized format; use a filename with a known extension", }, "suspicious": False, + "detection_status": "NOT_RUN", }, ) return @@ -520,18 +566,22 @@ def _handle_inspect(self, data: bytes, name: str, body: dict[str, Any]) -> None: report = inspect_image(path).to_dict() else: report = inspect_container(path).to_dict() - detected_wm = any( - entry.get("available") and entry.get("is_watermarked") - for entry in report.get("text_detectors") or [] - ) + detection_status = _text_detection_status(report.get("text_detectors") or []) suspicious = ( bool(report.get("suspicious_total")) or bool(report.get("has_c2pa") or report.get("has_ai_metadata")) or bool(report.get("stylometry", {}).get("score", 0.0) >= 0.65) - or detected_wm + or detection_status in {"DETECTED", "INCONCLUSIVE"} ) self._respond( - HTTPStatus.OK, {"ok": True, "kind": kind, "report": report, "suspicious": suspicious} + HTTPStatus.OK, + { + "ok": True, + "kind": kind, + "report": report, + "suspicious": suspicious, + "detection_status": detection_status, + }, ) def _handle_detect(self, data: bytes, name: str) -> None: diff --git a/skills/remove-ai-marks/scripts/text_detectors.py b/skills/remove-ai-marks/scripts/text_detectors.py index a392ef4..7955b2a 100644 --- a/skills/remove-ai-marks/scripts/text_detectors.py +++ b/skills/remove-ai-marks/scripts/text_detectors.py @@ -10,7 +10,10 @@ Reports follow the fail-soft contract: a detector that is unconfigured, times out, or errors returns {"available": False, "error": ...} and can -never block cleaning. +never block cleaning. Every report also carries "configured": whether the +detector was set up to run at all, so an aggregator can tell a detector that +never ran from one that ran and failed. A configured detector that failed is +unresolved evidence, never a clean result. Detectors: @@ -246,6 +249,7 @@ def detect(self, text: str) -> dict[str, Any]: "detector": self.name, "vendor": self.vendor, "available": False, + "configured": False, "error": "WATERMARKS_GEMINI_API_KEY not set", } @@ -255,6 +259,7 @@ def detect(self, text: str) -> dict[str, Any]: "detector": self.name, "vendor": self.vendor, "available": True, + "configured": True, "skipped": True, "reason": f"text longer than {max_chars} chars", "is_watermarked": None, @@ -264,6 +269,7 @@ def detect(self, text: str) -> dict[str, Any]: "detector": self.name, "vendor": self.vendor, "available": False, + "configured": False, "error": ( "DETECT_TEXT_WATERMARK task type is not supported by the " "Gemini generateContent API; this detector is disabled until " @@ -342,6 +348,7 @@ def detect(self, text: str) -> dict[str, Any]: "scheme": scheme, "vendor": "open-llm", "available": False, + "configured": bool(upstream), } if not upstream: report["error"] = "MARKLLM_DIR not set" @@ -441,6 +448,7 @@ def detect(self, text: str) -> dict[str, Any]: "detector": self.name, "vendor": self.vendor, "available": False, + "configured": False, "error": ( "Anthropic has announced a text-watermark detection API for " "Claude; no public endpoint is available yet. When it ships, " diff --git a/tests/test_http_server.py b/tests/test_http_server.py index da9be17..d518b3d 100644 --- a/tests/test_http_server.py +++ b/tests/test_http_server.py @@ -104,6 +104,7 @@ def test_openapi_spec_covers_all_endpoints(conn): "/openapi.json": {"get"}, "/inspect": {"post"}, "/clean": {"post"}, + "/detect": {"post"}, } for path, methods in expected.items(): assert path in body["paths"] @@ -122,6 +123,13 @@ def test_openapi_spec_describes_request_bodies(conn): assert "options" in schema["properties"] inspect = body["paths"]["/inspect"]["post"] assert "file" in inspect["requestBody"]["content"]["application/json"]["schema"]["properties"] + response = inspect["responses"]["200"]["content"]["application/json"]["schema"] + assert response["properties"]["detection_status"]["enum"] == [ + "DETECTED", + "INCONCLUSIVE", + "NOT_DETECTED", + "NOT_RUN", + ] def test_openapi_spec_reflects_auth(conn, monkeypatch): @@ -140,9 +148,91 @@ def test_inspect_text_finds_watermark(conn): assert status == 200 assert body["kind"] == "text" assert body["suspicious"] is True + assert body["detection_status"] == "NOT_RUN" assert body["report"]["suspicious_total"] >= 1 +def test_inspect_text_treats_inconclusive_detector_as_suspicious(conn, monkeypatch): + monkeypatch.setattr( + server, + "run_all_text_detectors", + lambda _text: [ + { + "detector": "markllm", + "available": True, + "verdict": "INCONCLUSIVE", + "is_watermarked": False, + } + ], + ) + status, body = _post( + conn, + "/inspect", + {"file": _b64(b"ordinary text"), "name": "note.txt", "detect": True}, + ) + assert status == 200 + assert body["suspicious"] is True + assert body["detection_status"] == "INCONCLUSIVE" + + +def test_inspect_text_treats_failed_configured_detector_as_suspicious(conn, monkeypatch): + """A configured detector that ran and failed is unresolved, not 'never ran'.""" + monkeypatch.setattr( + server, + "run_all_text_detectors", + lambda _text: [ + { + "detector": "markllm", + "available": False, + "configured": True, + "error": "MarkLLM detection timed out", + } + ], + ) + status, body = _post( + conn, + "/inspect", + {"file": _b64(b"ordinary text"), "name": "note.txt", "detect": True}, + ) + assert status == 200 + assert body["detection_status"] == "INCONCLUSIVE" + assert body["suspicious"] is True + + +@pytest.mark.parametrize( + ("reports", "expected"), + [ + ([{"available": True, "verdict": "DETECTED", "is_watermarked": True}], "DETECTED"), + ( + [ + {"available": True, "verdict": "INCONCLUSIVE", "is_watermarked": False}, + {"available": True, "verdict": "NOT_DETECTED", "is_watermarked": False}, + ], + "INCONCLUSIVE", + ), + ([{"available": True, "verdict": "UNSUPPORTED"}], "INCONCLUSIVE"), + ([{"available": True, "error": "detector failed"}], "INCONCLUSIVE"), + ([{"available": True, "verdict": "NOT_DETECTED", "is_watermarked": False}], "NOT_DETECTED"), + ([{"available": False, "configured": False, "error": "not configured"}], "NOT_RUN"), + ([{"available": False, "error": "not configured"}], "NOT_RUN"), + ( + [{"available": False, "configured": True, "error": "MarkLLM detection timed out"}], + "INCONCLUSIVE", + ), + ( + [ + {"available": False, "configured": True, "error": "MarkLLM detection timed out"}, + {"available": True, "verdict": "NOT_DETECTED", "is_watermarked": False}, + ], + "INCONCLUSIVE", + ), + ([], "NOT_RUN"), + ], +) +def test_text_detection_status(reports, expected): + assert server._text_detection_status(reports) == expected + + def test_clean_text_roundtrip(conn): data = "Hello​World­!".encode() status, body = _post(conn, "/clean", {"file": _b64(data), "name": "note.txt"}) @@ -220,6 +310,7 @@ def test_inspect_unknown_format_reports_kind(conn): assert status == 200 assert body["kind"] == "unknown" assert body["suspicious"] is False + assert body["detection_status"] == "NOT_RUN" assert "note" in body["report"] diff --git a/tests/test_markllm_detect.py b/tests/test_markllm_detect.py index 6a3136b..0701ddb 100644 --- a/tests/test_markllm_detect.py +++ b/tests/test_markllm_detect.py @@ -28,10 +28,14 @@ " print('MARKLLM_PRETRAINED_KWARGS=' + repr(kwargs), file=sys.stderr)\n" " return _LM()\n" "\n" + "class _Tok:\n" + " def encode(self, s):\n" + " return [0] * 512\n" + "\n" "class AutoTokenizer:\n" " @staticmethod\n" " def from_pretrained(name, **kwargs):\n" - " return object()\n" + " return _Tok()\n" ) FAKE_TRANSFORMERS_CONFIG = ( @@ -75,6 +79,23 @@ def _fake_auto_watermark(*, fail_detect: bool = False, fail_generate: bool = Fal ) +def _fake_auto_watermark_detect(detect_literal: str) -> str: + """Fake AutoWatermark returning a caller-supplied detect dict literal.""" + return ( + "from types import SimpleNamespace\n" + "class _WM:\n" + " def __init__(self):\n" + " self.config = SimpleNamespace(gen_kwargs={})\n" + " def detect_watermark(self, text, return_dict=True):\n" + f" return {detect_literal}\n" + "\n" + "class AutoWatermark:\n" + " @staticmethod\n" + " def load(algorithm_name, algorithm_config=None, transformers_config=None):\n" + " return _WM()\n" + ) + + def _make_fake_upstream( tmp_path: Path, *, @@ -82,6 +103,7 @@ def _make_fake_upstream( fail_detect: bool = False, fail_generate: bool = False, missing_watermark_dir: bool = False, + detect_literal: str | None = None, ) -> Path: upstream = tmp_path / "MarkLLM" config_dir = upstream / "config" @@ -93,9 +115,12 @@ def _make_fake_upstream( watermark = upstream / "watermark" watermark.mkdir(parents=True) (watermark / "__init__.py").write_text("") - (watermark / "auto_watermark.py").write_text( - _fake_auto_watermark(fail_detect=fail_detect, fail_generate=fail_generate) + fake = ( + _fake_auto_watermark_detect(detect_literal) + if detect_literal is not None + else _fake_auto_watermark(fail_detect=fail_detect, fail_generate=fail_generate) ) + (watermark / "auto_watermark.py").write_text(fake) utils_dir = upstream / "utils" utils_dir.mkdir(parents=True) (utils_dir / "__init__.py").write_text("") @@ -148,6 +173,47 @@ def test_cli_unavailable_missing_config(tmp_path: Path): assert "config" in (r.stderr or "").lower() +@pytest.mark.skipif(os.name != "posix" or os.geteuid() == 0, reason="needs POSIX perms") +def test_cli_unreadable_config_detect(tmp_path: Path): + """A config that becomes unreadable after resolution exits 3, not a traceback.""" + f = tmp_path / "t.txt" + f.write_text("hello world") + upstream = _make_fake_upstream(tmp_path) + config = upstream / "config" / "KGW.json" + config.chmod(0) + try: + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream)) + assert r.returncode == 3 + assert "cannot read watermarking config" in (r.stderr or "") + finally: + config.chmod(0o644) + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() == 0, reason="needs POSIX perms") +def test_cli_unreadable_config_watermark(tmp_path: Path): + """The watermark path reports the same unreadable-config failure, not a traceback.""" + prompt = tmp_path / "prompt.txt" + prompt.write_text("write about capybaras") + upstream = _make_fake_upstream(tmp_path) + config = upstream / "config" / "KGW.json" + config.chmod(0) + try: + r = _run_adapter( + "watermark", + str(prompt), + "--scheme", + "kgw", + "-o", + str(tmp_path / "wm.txt"), + "--upstream-dir", + str(upstream), + ) + assert r.returncode == 3 + assert "cannot read watermarking config" in (r.stderr or "") + finally: + config.chmod(0o644) + + def test_cli_unavailable_missing_deps(tmp_path: Path): upstream = tmp_path / "MarkLLM" (upstream / "config").mkdir(parents=True) @@ -393,6 +459,444 @@ def test_cli_watermark_json_stdout_stays_pure_without_output(tmp_path: Path): assert "WATERMARKED SAMPLE" not in (r.stdout or "") +def test_cli_detect_positive_verdict_detected(tmp_path: Path): + """A positive observation becomes DETECTED even with unknown provenance.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": True, "score": 9.2}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "DETECTED" + assert payload["detector_verdict"] == "DETECTED" + assert payload["is_watermarked"] is True + assert payload["provenance_match"] is None + assert payload["input_tokens"] == 512 + + +def test_cli_detect_positive_inside_abstention_band_stays_detected(tmp_path: Path): + """A positive observation is not downgraded by the abstention band.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": True, "score": 0.54}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "synthid-text", + "--upstream-dir", + str(upstream), + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "DETECTED" + assert payload["detector_verdict"] == "DETECTED" + assert payload["threshold"] == 0.52 + + +def test_cli_detect_positive_without_threshold_stays_detected(tmp_path: Path): + """A positive observation remains DETECTED without a config threshold.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": True, "score": 2.0}' + ) + config = tmp_path / "no-threshold.json" + config.write_text('{"algorithm_name": "KGW"}') + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--config", + str(config), + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "DETECTED" + assert payload["detector_verdict"] == "DETECTED" + assert payload["threshold"] is None + assert payload["verdict_reason"] == "detector reported watermarked" + + +def test_cli_detect_negative_no_key_inconclusive(tmp_path: Path): + """A negative with unknown provenance must NOT become 'clean'.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "INCONCLUSIVE" + assert payload["detector_verdict"] == "NOT_DETECTED" + assert payload["is_watermarked"] is False + assert payload["provenance_match"] is None + assert "provenance/key match not confirmed" in payload["verdict_reason"] + # Human output carries the warning line (non-JSON mode). + r2 = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream)) + assert r2.returncode == 0, r2.stderr + assert "does NOT establish that the document is watermark-free" in r2.stdout + + +def test_cli_detect_negative_with_key_not_detected(tmp_path: Path): + """A negative with an asserted key and enough tokens is a strong negative.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--key-id", + "kgw-test-v1", + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "NOT_DETECTED" + assert payload["provenance_match"] is True + assert payload["key_id"] == "kgw-test-v1" + + +def test_cli_detect_abstention_band_inconclusive(tmp_path: Path): + """A near-threshold score abstains even with a matching key.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.503}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "synthid-text", + "--upstream-dir", + str(upstream), + "--key-id", + "synthid-test-v1", + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "INCONCLUSIVE" + assert payload["threshold"] == 0.52 + assert "abstention band" in payload["verdict_reason"] + + +def test_cli_detect_short_text_inconclusive(tmp_path: Path): + """Too few scored tokens abstains even with a matching key.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--key-id", + "kgw-test-v1", + "--min-tokens", + "1000", + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "INCONCLUSIVE" + assert "below minimum calibrated length" in payload["verdict_reason"] + + +def test_cli_detect_unsupported_sidecar_scheme(tmp_path: Path): + """A sidecar declaring an unknown scheme yields UNSUPPORTED, no model load.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": True, "score": 9.2}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + (tmp_path / "t.txt.wm.json").write_text( + json.dumps({"scheme": "anthropic-synthid", "key_id": "claude-key"}) + ) + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "UNSUPPORTED" + assert payload["provenance_match"] is False + assert "anthropic-synthid" in payload["verdict_reason"] + + +def test_cli_detect_sidecar_config_hash_match(tmp_path: Path): + """Sidecar config_hash matching the detected config confirms provenance.""" + import hashlib + + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + (tmp_path / "t.txt.wm.json").write_text( + json.dumps( + { + "scheme": "KGW", + "key_id": "kgw-test-v1", + "config_hash": hashlib.sha256(KGW_CONFIG.encode()).hexdigest(), + } + ) + ) + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "NOT_DETECTED" + assert payload["provenance_match"] is True + assert payload["key_id"] == "kgw-test-v1" + + +def test_cli_detect_sidecar_config_hash_mismatch(tmp_path: Path): + """Sidecar config_hash differing from the config is a provenance mismatch.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + (tmp_path / "t.txt.wm.json").write_text( + json.dumps({"scheme": "KGW", "key_id": "kgw-test-v1", "config_hash": "deadbeef"}) + ) + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "INCONCLUSIVE" + assert payload["provenance_match"] is False + + +def test_cli_detect_sidecar_different_scheme_mismatch(tmp_path: Path): + """A sidecar declaring a different supported scheme is a mismatch.""" + import hashlib + + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + (tmp_path / "t.txt.wm.json").write_text( + json.dumps( + { + "scheme": "SynthID", + "key_id": "synthid-key", + "config_hash": hashlib.sha256(SYNTHID_CONFIG.encode()).hexdigest(), + } + ) + ) + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "INCONCLUSIVE" + assert payload["provenance_match"] is False + + +def test_cli_detect_sidecar_key_id_alone_stays_inconclusive(tmp_path: Path): + """A document-supplied key_id (no config_hash) must not grant a clean result.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + (tmp_path / "t.txt.wm.json").write_text(json.dumps({"scheme": "KGW", "key_id": "attacker"})) + r = _run_adapter("detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream), "--json") + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "INCONCLUSIVE" + assert payload["provenance_match"] is None + assert "provenance/key match not confirmed" in payload["verdict_reason"] + + +def test_cli_detect_few_scored_tokens_inconclusive(tmp_path: Path): + """The length gate reads the detector's scored-token count, not the raw input.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25, "num_tokens": 5}' + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--key-id", + "kgw-test-v1", + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + # The fake tokenizer reports 512 input tokens, so only the scored count + # (5) can trip the default 200-token minimum. + assert payload["input_tokens"] == 512 + assert payload["effective_scored_tokens"] == 5 + assert payload["verdict"] == "INCONCLUSIVE" + assert "below minimum calibrated length (5 < 200 tokens)" in payload["verdict_reason"] + + +def test_cli_detect_unknown_scored_tokens_inconclusive(tmp_path: Path): + """An unknown scored-token count must not fall through to a strong negative.""" + upstream = _make_fake_upstream( + tmp_path, detect_literal='{"is_watermarked": False, "score": 0.25}' + ) + # A tokenizer without .encode() makes both counts unknown. + (upstream / "transformers" / "__init__.py").write_text( + FAKE_TRANSFORMERS.replace( + " def encode(self, s):\n return [0] * 512\n", " pass\n" + ) + ) + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--key-id", + "kgw-test-v1", + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["input_tokens"] is None + assert payload["effective_scored_tokens"] is None + assert payload["verdict"] == "INCONCLUSIVE" + assert "scored token count unknown" in payload["verdict_reason"] + + +def test_cli_detect_missing_score_is_error(tmp_path: Path): + """A negative observation with no usable score is an ERROR, never clean.""" + upstream = _make_fake_upstream(tmp_path, detect_literal='{"is_watermarked": False}') + f = tmp_path / "t.txt" + f.write_text("hello world") + r = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--key-id", + "kgw-test-v1", + "--json", + ) + assert r.returncode == 0, r.stderr + payload = json.loads(r.stdout) + assert payload["verdict"] == "ERROR" + assert payload["detector_verdict"] == "NOT_DETECTED" + assert payload["score"] is None + assert "no score or threshold" in payload["verdict_reason"] + r2 = _run_adapter( + "detect", + str(f), + "--scheme", + "kgw", + "--upstream-dir", + str(upstream), + "--key-id", + "kgw-test-v1", + ) + assert r2.returncode == 0, r2.stderr + assert "does NOT establish that the document is watermark-free" in r2.stdout + + +def test_cli_watermark_writes_sidecar(tmp_path: Path): + """watermark -o writes a provenance sidecar next to the sample.""" + upstream = _make_fake_upstream(tmp_path) + prompt = tmp_path / "prompt.txt" + prompt.write_text("write about capybaras") + wm_out = tmp_path / "wm.txt" + r = _run_adapter( + "watermark", + str(prompt), + "--scheme", + "kgw", + "-o", + str(wm_out), + "--key-id", + "kgw-test-v1", + "--upstream-dir", + str(upstream), + "--device", + "cpu", + "--json", + ) + assert r.returncode == 0, r.stderr + sidecar = json.loads((tmp_path / "wm.txt.wm.json").read_text()) + assert sidecar["scheme"] == "KGW" + assert sidecar["key_id"] == "kgw-test-v1" + assert sidecar["config_hash"] + assert "implementation_commit" in sidecar # fake upstream has no git + assert "timestamp" in sidecar + assert sidecar["watermark_parameters"] == {"algorithm_name": "KGW", "z_threshold": 4.0} + + +def test_cli_watermark_sidecar_redacts_secret_config_fields(tmp_path: Path): + """The sidecar travels with the sample, so it must carry no key material.""" + upstream = _make_fake_upstream(tmp_path) + secret_config = tmp_path / "secret_kgw.json" + secret_config.write_text( + json.dumps( + { + "algorithm_name": "KGW", + "z_threshold": 4.0, + "hash_key": 15485863, + "nested": {"prf_keys": [1, 2, 3], "gamma": 0.5}, + } + ) + ) + prompt = tmp_path / "prompt.txt" + prompt.write_text("write about capybaras") + wm_out = tmp_path / "wm.txt" + r = _run_adapter( + "watermark", + str(prompt), + "--scheme", + "kgw", + "-o", + str(wm_out), + "--config", + str(secret_config), + "--upstream-dir", + str(upstream), + "--device", + "cpu", + "--json", + ) + assert r.returncode == 0, r.stderr + raw = (tmp_path / "wm.txt.wm.json").read_text() + assert "15485863" not in raw + sidecar = json.loads(raw) + params = sidecar["watermark_parameters"] + assert params["hash_key"] == "[redacted]" + assert params["nested"]["prf_keys"] == "[redacted]" + assert params["nested"]["gamma"] == 0.5 + assert params["z_threshold"] == 4.0 + assert sidecar["config_hash"] + + def test_cli_detect_applies_rlimit_as_in_child(tmp_path: Path): """--rlimit-as must cap the child's address space before heavy imports.""" if os.name == "nt": diff --git a/tests/test_text_detectors.py b/tests/test_text_detectors.py index f80a91b..26b06d1 100644 --- a/tests/test_text_detectors.py +++ b/tests/test_text_detectors.py @@ -125,6 +125,7 @@ def test_gemini_oversize_skips(monkeypatch): monkeypatch.setenv("WATERMARKS_GEMINI_MAX_CHARS", "10") report = text_detectors.GeminiSynthIDTextDetector().detect("x" * 100) assert report["available"] is True + assert report["configured"] is True assert report["skipped"] is True assert report["is_watermarked"] is None