diff --git a/STALE_SERVICES.txt b/STALE_SERVICES.txt new file mode 100644 index 0000000..b1965f9 --- /dev/null +++ b/STALE_SERVICES.txt @@ -0,0 +1,4 @@ +Not yet run. This file is overwritten by scripts/deploy_verify.py (the +`deploy-verify` compose service) on every `docker compose up` -- see that +service's definition in docker-compose.yml and the script's own docstring +for what gets checked and why. diff --git a/docker-compose.yml b/docker-compose.yml index 1047018..51deb91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,38 @@ services: timeout: 3s retries: 30 + # Deploy-freshness check (issue #200/#194 pattern -- a container running + # stale code relative to what's merged/available upstream, invisible + # until someone happens to test the exact behavior that changed; hit 3x + # in one session: celery-worker's rename regression, workbench's DAG- + # count regression, toolserver's 91-day-stale ghcr.io image). Runs once, + # compares each covered service's built/pulled image against what it + # should reflect, writes findings to stdout + STALE_SERVICES.txt. Never + # blocks startup -- see scripts/deploy_verify.py's own docstring for the + # full design. Every service it covers depends on this one with + # condition: service_completed_successfully below, so it runs + # automatically on every `docker compose up`, including + # `docker compose up -d ` -- no new command to remember. + deploy-verify: + image: python:3.12-slim + environment: + COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-omnibioai-studio} + command: + - bash + - -c + - | + pip install --no-cache-dir --quiet docker requests pyyaml && \ + python /workspace-config/scripts/deploy_verify.py + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./docker-compose.yml:/workspace-config/docker-compose.yml:ro + - ./.env:/workspace-config/.env:ro + - ./scripts/deploy_verify.py:/workspace-config/scripts/deploy_verify.py:ro + - ./STALE_SERVICES.txt:/workspace-config/STALE_SERVICES.txt:rw + - ${MACHINE_DIR}/omnibioai/.git:/repos/omnibioai:ro + - ${MACHINE_DIR}/omnibioai-tes/.git:/repos/omnibioai-tes:ro + restart: "no" + toolserver: image: ghcr.io/omnibioai/omnibioai-toolserver:latest restart: on-failure @@ -39,11 +71,14 @@ services: PYTHONUNBUFFERED: "1" IAM_URL: http://auth-service:8001 AUDIT_URL: http://security-audit:8004 + # deploy-verify default (14d) is fine for this one -- no override. volumes: - ${WORKSPACE_HOST}:/workspace depends_on: mysql: condition: service_healthy + deploy-verify: + condition: service_completed_successfully tes: image: omnibioai-tes-local @@ -124,6 +159,8 @@ services: condition: service_healthy toolserver: condition: service_started + deploy-verify: + condition: service_completed_successfully model-registry: build: @@ -435,6 +472,8 @@ services: condition: service_started model-registry: condition: service_started + deploy-verify: + condition: service_completed_successfully command: ["bash", "-lc", "python manage.py migrate || (python manage.py migrate --fake-initial && python manage.py migrate) && python manage.py runserver 0.0.0.0:8000 --noreload"] celery-worker: @@ -482,6 +521,8 @@ services: depends_on: workbench: condition: service_started + deploy-verify: + condition: service_completed_successfully command: ["bash", "-lc", "celery -A omnibioai worker -l info"] control-center: diff --git a/scripts/deploy_verify.py b/scripts/deploy_verify.py new file mode 100644 index 0000000..be3a870 --- /dev/null +++ b/scripts/deploy_verify.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +""" +scripts/deploy_verify.py -- deploy-freshness check (issue #200/#194 pattern: +a container running stale code relative to what's merged/available upstream, +invisible until someone happens to test the exact behavior that changed -- +hit 3x in one session: celery-worker rename regression, workbench DAG-count +regression, toolserver's 91-day-stale ghcr.io image). + +Runs once, as the `deploy-verify` one-shot service in docker-compose.yml, +wired via `depends_on: deploy-verify: condition: service_completed_successfully` +on every service this check covers -- so it runs automatically on every +`docker compose up` (including `docker compose up -d `), with +no new command for anyone to learn or remember. Always exits 0: this is a +loud warning system, not a startup gate -- a hard block would let a broken +upstream CI (see toolserver) take down the entire stack for reasons that +have nothing to do with today's deploy. + +Two independent checks, both boiling down to "is this image older than the +thing it's supposed to reflect": + + - build-from-source services (workbench, celery-worker, tes, ...): + compare the image's own `Created` timestamp against origin/main's + latest commit timestamp (fetched via one lightweight GitHub REST API + call -- NOT `git fetch`, no object transfer, just commit metadata). + Also reads the local repo's `refs/heads/main` ref file directly (plain + text file, no `git` binary needed) to distinguish "forgot to pull" + (local ref behind origin) from "forgot to rebuild" (local ref matches + origin, image just wasn't rebuilt after). + + - pull-from-registry services (toolserver, ...): no source repo to + compare against by definition -- that's the actual gap. Falls back to + an age threshold: stale if the image is older than `max_age_days`. + Same age-threshold also used as build-type's fallback when the GitHub + API call fails (rate limit, network, token issue) -- never crashes, + never blocks, degrades to "best information available." + +Per-service `max_age_days` override: a `deploy-verify.max-age-days` label +in that service's docker-compose.yml `labels:` block (default 14 if +absent). Read directly from the raw compose YAML, not docker-compose's own +resolved config -- this container doesn't have the `docker compose` CLI +available, just the Docker socket. + +Output goes to BOTH stdout (visible in `docker compose up`'s own combined +output / `docker compose logs deploy-verify`) AND STALE_SERVICES.txt at +this repo's root (overwritten every run) -- redundant channels so a +staleness finding is harder to scroll past. +""" +from __future__ import annotations + +import os +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +import docker +import requests +import yaml + +COMPOSE_FILE = Path("/workspace-config/docker-compose.yml") +ENV_FILE = Path("/workspace-config/.env") +OUT_FILE = Path("/workspace-config/STALE_SERVICES.txt") +REPOS_ROOT = Path("/repos") # each covered repo's .git mounted read-only under here, see docker-compose.yml + +DEFAULT_MAX_AGE_DAYS = 14 +GITHUB_API = "https://api.github.com" + +# Services this check covers today -- the ones actually hit this session, +# plus their .git mount name under /repos. Deliberately a short, explicit +# list rather than trying to auto-derive "source repo" for every one of +# the ~30 services in this compose file (several build from a shared +# multi-repo context where that inference gets ambiguous) -- extending +# coverage to more services is a straightforward follow-up once this is +# validated, not a blocker for v1. +COVERED_SERVICES = { + "workbench": {"type": "build", "repo_mount": "omnibioai"}, + "celery-worker": {"type": "build", "repo_mount": "omnibioai"}, + "tes": {"type": "build", "repo_mount": "omnibioai-tes"}, + "toolserver": {"type": "pull"}, +} + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def load_env_file(path: Path) -> dict: + env = dict(os.environ) + if not path.exists(): + return env + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip()) + return env + + +_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(:-(.*?))?\}") + + +def resolve_vars(value: str, env: dict) -> str: + """Minimal ${VAR:-default} / ${VAR} resolution -- not full Compose + interpolation, just enough for the handful of fields this script reads + (image, labels).""" + if not isinstance(value, str): + return value + + def _sub(m: re.Match) -> str: + name, _, default = m.groups() + return env.get(name, default or "") + + return _VAR_RE.sub(_sub, value) + + +def max_age_days_for(service_cfg: dict, env: dict) -> int: + labels = service_cfg.get("labels") or [] + pairs = [] + if isinstance(labels, dict): + pairs = list(labels.items()) + elif isinstance(labels, list): + for entry in labels: + if "=" in entry: + k, v = entry.split("=", 1) + pairs.append((k, v)) + for k, v in pairs: + if k.strip() == "deploy-verify.max-age-days": + try: + return int(resolve_vars(str(v), env)) + except ValueError: + pass + return DEFAULT_MAX_AGE_DAYS + + +def image_created(client: "docker.DockerClient", image_ref: str): + """Returns (found: bool, created: datetime|None).""" + try: + img = client.images.get(image_ref) + except docker.errors.ImageNotFound: + return False, None + created_str = img.attrs.get("Created", "") + try: + # Python 3.11+'s fromisoformat handles 'Z' and any fractional-second + # precision natively (Docker's own precision varies) -- no manual + # normalization needed. + return True, datetime.fromisoformat(created_str) + except ValueError: + return True, None + + +def local_main_sha(repo_mount: str) -> str | None: + """Reads refs/heads/main directly -- plain-text file, no `git` binary + needed. Falls back to packed-refs if main was ever packed and no loose + ref file exists. Deliberately reads refs/heads/main specifically, not + HEAD -- HEAD may point at whatever branch happens to be checked out + locally (a feature branch mid-work), which is not what "is this image + stale relative to main" should be asking.""" + root = REPOS_ROOT / repo_mount + loose = root / "refs" / "heads" / "main" + if loose.exists(): + return loose.read_text().strip() + packed = root / "packed-refs" + if packed.exists(): + for line in packed.read_text().splitlines(): + if line.endswith("refs/heads/main"): + return line.split()[0] + return None + + +def remote_url_to_owner_repo(repo_mount: str) -> tuple[str, str] | None: + config = REPOS_ROOT / repo_mount / "config" + if not config.exists(): + return None + text = config.read_text() + m = re.search(r"url\s*=\s*.*[:/]([\w.-]+)/([\w.-]+?)(\.git)?\s*$", text, re.MULTILINE) + if not m: + return None + return m.group(1), m.group(2) + + +def github_main_commit(owner: str, repo: str, token: str): + """Single lightweight REST call for origin/main's tip -- not `git + fetch`: no object transfer, just commit metadata (sha + date). Returns + (sha, committed_at: datetime) or (None, None) on any failure -- never + raises, caller falls back to the age-threshold check.""" + try: + r = requests.get( + f"{GITHUB_API}/repos/{owner}/{repo}/commits/main", + headers={"Authorization": f"Bearer {token}"} if token else {}, + timeout=10, + ) + if r.status_code != 200: + return None, None + body = r.json() + sha = body.get("sha") + date_str = body.get("commit", {}).get("committer", {}).get("date") + committed_at = datetime.fromisoformat(date_str) if date_str else None + return sha, committed_at + except Exception: + return None, None + + +def check_build_service(name: str, cfg: dict, client, env: dict, token: str, lines: list[str]) -> bool: + """Returns True if OK, False if stale (or undetermined -- treated as + non-fatal but reported).""" + repo_mount = cfg["repo_mount"] + raw_image = cfg.get("_raw", {}).get("image") + if raw_image: + # Explicit `image:` override on an otherwise build-from-source + # service (e.g. tes's `image: omnibioai-tes-local`) -- use it + # instead of assuming Compose's auto-derived - + # tag, which only applies when no explicit image is set. + image_ref = resolve_vars(raw_image, env) + if ":" not in image_ref.split("/")[-1]: + image_ref += ":latest" + else: + project = env.get("COMPOSE_PROJECT_NAME", "omnibioai-studio") + image_ref = f"{project}-{name}:latest" + found, created = image_created(client, image_ref) + if not found: + lines.append(f"[SKIP] {name:16s} image {image_ref} not built yet -- nothing to compare") + return True + + local_sha = local_main_sha(repo_mount) + owner_repo = remote_url_to_owner_repo(repo_mount) + remote_sha, remote_date = (None, None) + if owner_repo: + remote_sha, remote_date = github_main_commit(*owner_repo, token) + + if remote_date is not None and created is not None: + if created >= remote_date: + lines.append(f"[OK] {name:16s} image built {created.isoformat()}, matches {repo_mount}@main HEAD ({remote_date.isoformat()})") + return True + behind_pull = local_sha is not None and remote_sha is not None and local_sha != remote_sha + reason = "pull AND rebuild needed (local checkout is also behind origin/main)" if behind_pull else "rebuild needed (local checkout matches origin/main; image predates it)" + lines.append( + f"[STALE] {name:16s} image built {created.isoformat()}; " + f"{repo_mount}@origin/main HEAD is {remote_date.isoformat()} ({(remote_date - created).days}d newer) -- {reason}" + ) + return False + + # GitHub API unreachable -- fall back to age threshold, degrade + # gracefully rather than skip the check entirely. + max_age = max_age_days_for(cfg.get("_raw", {}), env) + if created is None: + lines.append(f"[SKIP] {name:16s} could not read image build time or reach GitHub API -- no signal available") + return True + age_days = (datetime.now(timezone.utc) - created).days + if age_days > max_age: + lines.append(f"[STALE?] {name:16s} image is {age_days}d old (threshold {max_age}d) -- GitHub API unreachable, falling back to age check only") + return False + lines.append(f"[OK] {name:16s} image is {age_days}d old, within {max_age}d fallback threshold (GitHub API unreachable, could not compare to origin/main)") + return True + + +def check_pull_service(name: str, cfg: dict, client, env: dict, raw_svc: dict, lines: list[str]) -> bool: + image_ref = resolve_vars(raw_svc.get("image", ""), env) + if not image_ref: + lines.append(f"[SKIP] {name:16s} no resolvable image reference") + return True + found, created = image_created(client, image_ref) + if not found or created is None: + lines.append(f"[SKIP] {name:16s} image {image_ref} not pulled yet -- nothing to compare") + return True + max_age = max_age_days_for(raw_svc, env) + age_days = (datetime.now(timezone.utc) - created).days + if age_days > max_age: + lines.append(f"[STALE] {name:16s} image created {created.isoformat()} ({age_days}d old, threshold {max_age}d) -- registry may have stopped publishing, see #200") + return False + lines.append(f"[OK] {name:16s} image created {created.isoformat()} ({age_days}d old), within {max_age}d threshold") + return True + + +def main() -> int: + env = load_env_file(ENV_FILE) + token = env.get("GHCR_PULL_TOKEN", "") + + raw_compose = {} + if COMPOSE_FILE.exists(): + raw_compose = yaml.safe_load(COMPOSE_FILE.read_text()) or {} + raw_services = raw_compose.get("services", {}) + + client = docker.from_env() + + lines = ["==================== DEPLOY FRESHNESS CHECK ===================="] + any_stale = False + + for name, cfg in COVERED_SERVICES.items(): + raw_svc = raw_services.get(name, {}) + if cfg["type"] == "build": + cfg = {**cfg, "_raw": raw_svc} + ok = check_build_service(name, cfg, client, env, token, lines) + else: + ok = check_pull_service(name, cfg, client, env, raw_svc, lines) + any_stale = any_stale or not ok + + lines.append("===================================================================") + if any_stale: + lines.append("One or more services are stale -- see above. This check never blocks") + lines.append("startup; treat this as a loud reminder, not a gate.") + else: + lines.append("All covered services fresh.") + + output = "\n".join(lines) + log(output) + OUT_FILE.write_text(output + "\n") + + return 0 # always -- see module docstring + + +if __name__ == "__main__": + sys.exit(main())