Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions .github/workflows/pip-audit-optional.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: pip-audit (optional backends)

on:
push:
branches: [main]
paths:
- "skills/remove-ai-marks/scripts/requirements-*.txt"
- ".github/workflows/pip-audit-optional.yml"
pull_request:
paths:
- "skills/remove-ai-marks/scripts/requirements-*.txt"
- ".github/workflows/pip-audit-optional.yml"
schedule:
# Weekly Monday report so dismissed Dependabot alerts stay visible.
- cron: "17 4 * * 1"
workflow_dispatch:

permissions:
contents: read

jobs:
audit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Comment thread
elkaix marked this conversation as resolved.
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"

- name: Install pip-audit
run: python -m pip install --upgrade pip pip-audit==2.10.1

- name: Audit optional-backend requirement files
# Report-only by design: the audited files pin research-backend versions
# deliberately (see skills/remove-ai-marks/scripts/requirements-ctrlregen.txt
# header and research/dependabot-ctrlregen-advisory-review.md). Findings must
# surface here without blocking CI, while checkout/setup/install failures
# above stay fatal.
continue-on-error: true
run: |
set +e
rc=0
for f in skills/remove-ai-marks/scripts/requirements-*.txt; do
echo "::group::pip-audit $f"
python -m pip_audit -r "$f" --progress-spinner off || rc=1
echo "::endgroup::"
done
echo "pip-audit sweep finished (rc=${rc}, report-only)"
exit "$rc"
52 changes: 48 additions & 4 deletions skills/remove-ai-marks/scripts/markdiffusion_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import argparse
import io
import os
import re
import sys
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -60,6 +61,10 @@
IMAGE_SCHEMES = {"TR", "RI", "ROBIN", "WIND", "SFW", "GS", "GM", "PRC", "SEAL"}

DEFAULT_MODEL = "huanzi05/stable-diffusion-2-1-base"
# Hub HEAD of DEFAULT_MODEL (verified via the HF API in 2026-08), pinned so the
# default load is reproducible. Bump deliberately, never automatically.
DEFAULT_MODEL_REVISION = "f71d7867a2745c420aa93441638b119c85995963"
_DEFAULT_PINNED_MODEL = f"{DEFAULT_MODEL}@{DEFAULT_MODEL_REVISION}"

# Algorithm configs are a few hundred bytes (TR.json/GS.json). Cap well above
# that so a crafted or accidental huge file is refused before either this script
Expand Down Expand Up @@ -122,21 +127,46 @@ def _import_markdiffusion(upstream: Path | None) -> Any:
return markdiffusion


_FULL_COMMIT_SHA = re.compile(r"\A[0-9a-f]{40}\Z")


def _split_model_revision(model: str) -> tuple[str, str | None]:
"""Split a mandatory ``org/repo@<full-commit-sha>`` suffix from a --model value.

Pinning a revision keeps Hub loads reproducible and shrinks the
malicious-repository swap surface exposed by CVE-2026-44513-class
diffusers supply-chain attacks, so mutable refs (branches, tags) are
rejected here and unrevisioned models must go through --offline.
"""
repo, sep, revision = model.partition("@")
if sep and (not repo or not revision):
raise ValueError(f"invalid model {model!r}: expected 'org/repo@<full-commit-sha>'")
if sep and not _FULL_COMMIT_SHA.fullmatch(revision):
raise ValueError(
f"invalid revision {revision!r} in {model!r}: pass the full 40-character "
"commit ID (branches and tags are mutable refs)"
)
return (repo, revision) if sep else (model, None)
Comment thread
elkaix marked this conversation as resolved.


def _load_diffusion(model: str, device: str, offline: bool, size: int):
"""Load the Stable Diffusion pipeline and scheduler used by the harness."""
if offline:
os.environ.setdefault("HF_HUB_OFFLINE", "1")
load_kwargs = {"local_files_only": True} if offline else {}
repo, revision = _split_model_revision(model)
if revision is not None:
load_kwargs["revision"] = revision

import torch
from diffusers import DPMSolverMultistepScheduler, StableDiffusionPipeline

scheduler = DPMSolverMultistepScheduler.from_pretrained(
model, subfolder="scheduler", **load_kwargs
repo, subfolder="scheduler", **load_kwargs
)
dtype = torch.float16 if device == "cuda" else torch.float32
pipe = StableDiffusionPipeline.from_pretrained(
model,
repo,
scheduler=scheduler,
torch_dtype=dtype,
safety_checker=None,
Expand Down Expand Up @@ -426,8 +456,10 @@ def _add_common(p: argparse.ArgumentParser) -> None:
)
p.add_argument(
"--model",
default=os.environ.get("MARKDIFFUSION_MODEL", DEFAULT_MODEL),
help=f"HF Stable Diffusion model (default: $MARKDIFFUSION_MODEL or {DEFAULT_MODEL})",
default=os.environ.get("MARKDIFFUSION_MODEL", _DEFAULT_PINNED_MODEL),
help="HF Stable Diffusion model as org/repo@<full-commit-sha>; mutable refs "
"are rejected unless --offline "
f"(default: $MARKDIFFUSION_MODEL or {_DEFAULT_PINNED_MODEL})",
)
p.add_argument(
"--device",
Expand Down Expand Up @@ -521,6 +553,18 @@ def main() -> int:
eprint(str(e))
return 2

try:
_, revision = _split_model_revision(args.model)
except ValueError as e:
eprint(str(e))
return 2
if revision is None and not args.offline:
eprint(
f"unpinned model {args.model!r}: pass org/repo@<full-commit-sha> "
"(or use --offline to load from the local cache only)"
)
return 2

raw_upstream = args.upstream_dir or os.environ.get("MARKDIFFUSION_DIR")
upstream = resolve_upstream(str(raw_upstream) if raw_upstream else None)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,9 @@
# platform index (CUDA or CPU) and satisfies markdiffusion's own
# torch>=2.4,<2.11 range, so it is intentionally not listed here.
markdiffusion==1.0.2

# diffusers is pinned explicitly because markdiffusion declares only
# diffusers>=0.25; unpinned, pip resolves latest and load/audit behavior
# drifts between runs. Bump deliberately after validating against the pinned
# markdiffusion release.
diffusers==0.40.0
64 changes: 64 additions & 0 deletions tests/test_markdiffusion_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,67 @@ def test_cli_resolve_config_missing(tmp_path: Path):

with __import__("pytest").raises(_Unavailable):
_resolve_config(None, str(tmp_path / "missing.json"))


_VALID_SHA = "f71d7867a2745c420aa93441638b119c85995963"


def test_split_model_revision_accepts_full_commit_sha():
"""A full 40-char lowercase hex commit ID passes through verbatim."""
from markdiffusion_harness import _split_model_revision

assert _split_model_revision(f"org/repo@{_VALID_SHA}") == ("org/repo", _VALID_SHA)


def test_split_model_revision_unrevisioned_returns_none():
"""No '@' -> repo id unchanged and revision None."""
from markdiffusion_harness import _split_model_revision

assert _split_model_revision("org/repo") == ("org/repo", None)


def test_split_model_revision_rejects_mutable_refs_and_malformed():
"""Branches, tags, short/long/non-hex SHAs and malformed specs raise."""
import pytest
from markdiffusion_harness import _split_model_revision

for bad in (
"org/repo@main",
"org/repo@v1.0.0",
"org/repo@" + "a" * 39,
"org/repo@" + "a" * 41,
"org/repo@" + "g" * 40,
"@" + "a" * 40,
"org/repo@",
):
with pytest.raises(ValueError):
_split_model_revision(bad)


def test_default_model_is_pinned_to_full_sha():
"""The built-in default carries an immutable full-commit-SHA revision."""
from markdiffusion_harness import (
DEFAULT_MODEL,
DEFAULT_MODEL_REVISION,
_split_model_revision,
)

repo, revision = _split_model_revision(f"{DEFAULT_MODEL}@{DEFAULT_MODEL_REVISION}")
assert repo == DEFAULT_MODEL
assert revision == DEFAULT_MODEL_REVISION


def test_cli_unpinned_online_model_rejected(tmp_path: Path):
"""Online + unrevisioned --model -> exit 2 before any upstream access."""
img = tmp_path / "img.png"
img.write_bytes(b"x")
r = _run_adapter(
"detect",
str(img),
"--scheme",
"tr",
"--model",
"huanzi05/stable-diffusion-2-1-base",
)
assert r.returncode == 2
assert "unpinned" in (r.stderr or "")
Loading