diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f3a716c..b92471f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,19 +4,28 @@ updates: directory: "/" schedule: interval: "weekly" - open-pull-requests-limit: 5 + day: "monday" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "python" groups: minor-and-patch: update-types: - "minor" - "patch" + - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" + day: "monday" open-pull-requests-limit: 5 + labels: + - "dependencies" + - "github-actions" groups: minor-and-patch: update-types: - "minor" - - "patch" \ No newline at end of file + - "patch" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fde4b37..9752377 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,8 @@ jobs: PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: "3.13" - name: Install diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 04edc42..ab0165b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -20,19 +20,21 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'python' ] + include: + - language: python + build-mode: none steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + build-mode: ${{ matrix.build-mode }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml new file mode 100644 index 0000000..b88d133 --- /dev/null +++ b/.github/workflows/gitleaks.yml @@ -0,0 +1,30 @@ +# NOTE: requires gitleaks/gitleaks-action to be allowlisted under +# Settings -> Actions -> General -> Actions permissions (repo currently +# restricts to selected/GitHub-owned actions only, or "Allow all actions"). +# Otherwise the run ends in startup_failure with zero jobs. +name: gitleaks + +on: + push: + branches: [master, dev] + pull_request: + branches: [master, dev] + workflow_dispatch: + +permissions: + contents: read + +jobs: + scan: + name: gitleaks + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: gitleaks/gitleaks-action@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..47d410a --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,28 @@ +# NOTE: requires googleapis/release-please-action to be allowlisted under +# Settings -> Actions -> General -> Actions permissions (repo currently +# restricts to selected/GitHub-owned actions only, or "Allow all actions"). +# Otherwise the run ends in startup_failure with zero jobs. +# +# Operator note: this workflow authenticates with the default +# secrets.GITHUB_TOKEN. GitHub suppresses pull_request-triggered checks +# (CI/CodeQL) on PRs opened by the default GITHUB_TOKEN, so if branch +# protection on master later requires those checks to pass, the +# release-please PR will need a fine-grained PAT / GitHub App token +# instead of GITHUB_TOKEN to trigger them normally. +name: release-please + +on: + push: + branches: [master] + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: googleapis/release-please-action@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.planning/.continue-here.md b/.planning/.continue-here.md index 82e93cf..a7f022e 100644 --- a/.planning/.continue-here.md +++ b/.planning/.continue-here.md @@ -1,79 +1,149 @@ --- -context: research -milestone: v3.0 Resilience + Ecosystem -workflow: new-milestone -stage: research-synthesis-incomplete +context: default +phase: none (milestone v4.2 complete + archived; post-milestone UAT audit session) +task: none +total_tasks: 0 status: paused -last_updated: 2026-06-07T08:13:27.961Z +last_updated: 2026-08-01T23:44:34.447Z --- -# BLOCKING CONSTRAINTS — Read Before Anything Else - -> Discovered through failure this session. Acknowledge before proceeding. - -- [ ] CONSTRAINT: `.planning/research/SUMMARY.md` is STALE v1 content (Selenium→nodriver, Phase 1). The v3.0 synthesizer errored before writing it. Do NOT read SUMMARY.md as v3.0 research. Mitigation: regenerate it via `gsd-research-synthesizer` from the 4 fresh v3.0 files BEFORE defining requirements. - -**Do not proceed until the box is checked.** - ## Critical Anti-Patterns | Pattern | Description | Severity | Prevention Mechanism | |---------|-------------|----------|----------------------| -| Stale-SUMMARY consumption | SUMMARY.md on disk is the v1 summary, not v3.0. Reading it would ground v3.0 requirements in the wrong milestone. | blocking | First resume action = re-spawn synthesizer to overwrite SUMMARY.md; verify its first lines reference v3.0 / proxy-CAPTCHA-price before using it. | +| `gsd-sdk query audit-uat` false all-clear | It scans `.planning/phases/`, which is EMPTY once milestones are archived. It reported `total_items: 0` while **80** real outstanding items sat in `.planning/milestones/*-phases/`. Following its "All Clear / stop here" branch would have closed 79 live items on a tooling blind spot. | blocking | Never trust the SDK's UAT all-clear after an archive. Glob `.planning/**/*UAT*.md` and `.planning/**/*VERIFICATION*.md` directly and check the archive dirs yourself. | +| `rtk pytest` wrong interpreter | On this machine `rtk pytest` / bare `pytest` resolve to a stray Python 3.14 with a corrupted `nodriver` package → ~24 false collection errors | advisory | Always run `.venv\Scripts\python.exe -m pytest -q` (project venv, Python 3.13). `httpx` is now declared in `requirements.txt`, so the manual install is no longer needed. | +| STATE.md schema drift breaks SDK verbs | STATE.md uses a custom prose schema; `gsd-sdk state.record-session` / `advance-plan` / `roadmap.update-plan-progress` silently no-op or error on it | advisory | Hand-edit STATE.md to match the existing convention when an SDK state verb fails. | +| Module-scope test imports break under `sys.modules` purge | `tests/test_cli_no_fastapi.py:17-19` deletes `core.cli.web` from `sys.modules` to force its lazy-import guard. Any test file that imports from that module at module scope binds a DEAD module object: tests pass in isolation and FAIL in a full-suite run. Cost a real debug cycle. | advisory | In tests touching `core.cli.web` (or any purged module), resolve the module INSIDE each test body and monkeypatch the module object, not a dotted string path. See `tests/test_cli_web_port.py`. | +| Uncommitted work in the session scratchpad | The scratchpad directory (`%TEMP%\claude\...\scratchpad`) was wiped mid-session by something outside the session, destroying two git worktrees and their uncommitted edits. | advisory | Never place a git worktree or any uncommitted work under the scratchpad. Use a durable path such as `E:\repos\ShopPyBot-wt-`. | +| Auto-mode classifier blocks remote mutations | `gh pr merge`, `gh api -X PUT .../branches/master/protection`, and `Write` outside the project dir are all DENIED even with explicit in-chat user approval. | advisory | Do not retry in a different shell to route around it. Hand the operator the exact command to run with the `!` prefix. Also: `gh pr comment --body-file -` (stdin) fails through rtk with "Body cannot be blank" — write the body to a file and pass its path. | -Mid `/gsd:new-milestone` for **v3.0 Resilience + Ecosystem**. Milestone is started: PROJECT.md has the Current Milestone v3.0 section, STATE.md switched to v3.0 (status=planning), both committed (e159f24). `.planning/phases/` is empty (cleared). `.planning/REQUIREMENTS.md` does NOT exist yet (removed at v2.0 close; new-milestone recreates it). +Milestone **v4.2 is COMPLETE and archived**; there is no in-progress GSD phase work. This +session was a post-milestone cross-phase UAT audit that turned up three real defects. -Research-first was chosen. The 4 research agents (Stack/Features/Architecture/Pitfalls) finished and wrote solid v3.0 files. The synthesizer (SUMMARY.md) errored + was stopped — so SUMMARY.md is still the old v1 file. +Working tree is CLEAN. Four commits are local-only on `chore/v4.0-milestone-close` +(`d485710`, `0cebc9e`, `70f31c5`, `f883f13`) and have not been pushed. + +`master` has moved: PR #13 merged as `4123059`. **CI is green for the first time in the +project's history** — 755 passed / 2 skipped, identical on `ubuntu-latest` and +`windows-latest`. Branch protection now requires contexts that actually exist. + +PR #12 is rebased, all four checks green, `MERGEABLE` / `CLEAN`, awaiting an operator merge. -- v3.0 scope gathered: all four areas, FULL (anti-detection hardening, plugin ecosystem, price monitoring, stability/polish) -- PROJECT.md updated (Current Milestone v3.0 + Active reqs) — committed e159f24 -- STATE.md switched to v3.0 — committed e159f24 -- Research: STACK.md, FEATURES.md, ARCHITECTURE.md, PITFALLS.md written (committed in this WIP) +- Cross-phase UAT audit: 80 outstanding items across 66 files, found in the archive dirs +- Claude-in-Chrome handoff doc authored, delivered, and executed +- Live browser UAT sweep: **13 PASS / 1 FAIL / 5 BLOCKED / 2 PARTIAL** +- SEED-003 planted (`d485710`): remote plugin manager, third-party disclaimer, extensibility +- Port auto-select for `shoppybot web` + 18 tests (`0cebc9e`) +- UAT verdicts recorded into 11 phase artifacts (`70f31c5`), +246/-40, frontmatter validated +- STATE.md session record (`f883f13`) +- **PR #13 MERGED** (`4123059`): CI compile fix + dependency install + `httpx` +- **PR #12 open and green**: `_register_signals` main-thread guard + 4 regression tests +- Operator closed: A1 PVR, A2 Actions allowlist, A4 CodeQL, A6 wiki, A7 sign-off, branch protection +- `31-HV-1` (gitleaks) and `31-HV-2` (CodeQL) flipped to PASS — first greens ever for both -1. **Synthesize v3.0 SUMMARY.md** — re-spawn `gsd-research-synthesizer` (sonnet) over the 4 files; overwrite the stale SUMMARY. -2. **Define requirements** (new-milestone.md step 9) — scope each area via AskUserQuestion multiselect; write `.planning/REQUIREMENTS.md` with new REQ-IDs (continue ANTI-04+; new categories e.g. PRICE-/ECO-/STAB-); commit. -3. **Create roadmap** (step 10) — spawn `gsd-roadmapper`, **start phase numbering at 12**. Research build order: 12 anti-detect infra → 13 price (parallelizable) → 14 ecosystem/metadata → 15 BotService+front-ends → 16 stability. Approve, commit. -4. **Link pending todos** (step 10.5), then done banner. +1. Merge PR #12 (operator; classifier blocks agent merges) +2. Push the 4 local commits on `chore/v4.0-milestone-close` +3. Decide on PR #11 (264 commits) +4. Unpause Dependabot, then re-check `31-HV-3` (7 alerts, #6-#12) +5. Re-test `27-HV-2`, `28-HV-1`, `28-HV-2` running-half once PR #12 lands +6. `10-HV-3` / `MC-4`: needs a `--host 0.0.0.0` restart +7. `29-HV-5`: needs a DevTools source breakpoint +8. `MC-1` / `MC-2`: Windows TTY checks, the only two items with zero prerequisites +9. Repair 12 drifted UAT docs — three produce FALSE PASSES as written +10. Fix the `docs/PLUGIN_REGISTRY.md` example row (`hard` vs the inherited `medium`) -- v3.0 = all four areas, FULL (major release). -- Research-first. -- Only ONE new dep across the milestone: `2captcha-python 2.0.7` (AsyncTwoCaptcha, MIT). Proxy/fingerprint/price/wiki = zero new deps. -- Walmart/Target use behavioral bot managers (HUMAN/Akamai), NOT CAPTCHA-solvable — keep manual-pause for those. -- `core/browser_factory.py` = shared hook for proxy/fingerprint/CAPTCHA; ABC `setup()` default calls it; per-plugin migration, zero forced rewrites of the 7 plugins. -- New secrets via CredentialStore SECRET_KEYS (TWOCAPTCHA_API_KEY, CAPSOLVER_API_KEY, PROXY_USERNAME, PROXY_PASSWORD); CAPTCHA opt-in default false (like SMS gate); price-drop dedup uses SEPARATE DB columns from stock dedup. +- Wiki Plugin Registry published **headers-only**: `last-verified` means confirmed against a + live retail site, no plugin ever has been (`19-UAT-1`), so any date would be fabricated +- **PVR is the sole security-report channel**; no maintainer email published (`32-HV-1`) +- `_register_signals` fixed with a **main-thread guard**, not a widened `except`: signal + disposition is process-level and main-thread-only on both platforms, and the threaded path + already has `BotService.stop()` for cooperative cancellation +- Branch protection requires `test (ubuntu-latest)`, `test (windows-latest)`, `CodeQL`. + Kept the stable `CodeQL` aggregate over `Analyze (python)`, which PR #11 renames to + `Analyze (python, none)` via `build-mode` — requiring the old name would permanently block +- `httpx==0.28.1` added to `requirements.txt` rather than a new extra -- v3.0 SUMMARY.md not generated (synthesizer errored). Workaround: regenerate first thing on resume. +- **Agent cannot merge PRs or write branch protection** — operator runs them with `!` +- **`31-HV-3` will not drain** — two causes: remediating bumps live on + `chore/v4.0-milestone-close` (needs PR #11), and Dependabot is repo-level PAUSED +- **PR #11 `mergeStateStatus` reads UNKNOWN** — re-query after PR #12 lands ## Required Reading (in order) -1. `.planning/HANDOFF.json` — structured state mirror of this file. -2. `.planning/research/STACK.md`, `FEATURES.md`, `ARCHITECTURE.md`, `PITFALLS.md` — the real v3.0 research (NOT SUMMARY.md until regenerated). -3. `$HOME/.claude/get-shit-done/workflows/new-milestone.md` — resume at step 9. -## Critical Anti-Patterns (do NOT repeat these) -- Stale-SUMMARY consumption → regenerate SUMMARY.md before reading it (see blocking table above). +1. `.planning/STATE.md` → "UAT Audit Session — 2026-08-01" — full verdict table, the three + defects, operator actions closed, outstanding items, and the documentation-drift list +2. `.planning/HANDOFF.json` — machine-readable state for `/gsd:resume-work` +3. `.planning/seeds/SEED-003-*.md` — will surface at the next `/gsd:new-milestone` +4. Phase artifacts under `.planning/milestones/*/` — each now carries real verdicts with + evidence instead of `[pending]` + +## The three defects (context for anyone resuming) + +None were caught by any test or milestone review. All three were invisible for the same +reason: nothing exercised them. + +1. **Dashboard Start Bot never worked.** `core/orchestrator.py:_register_signals` calls + `signal.signal()` off the main thread; `BotService.start()` runs `async_main` in a daemon + thread, so it raised `ValueError` at `async_main`'s fifth statement and the bot loop died + before plugin setup — while `POST /api/bot/start` still returned 200 and the UI showed + nothing. The CLI path masked it by running on the main thread. Fixed in PR #12. +2. **CI never compiled.** `${{ runner.temp }}` in job-level `env:`, where the `runner` + context does not exist. 81 runs, 81 failures, zero jobs scheduled, no logs — it reads like + a flaky runner, not a syntax error. Introduced by `0e0e43f`, the only commit that ever + touched the file. Fixed in PR #13. +3. **CI never installed dependencies.** The install step ran only `pip install -e .[web]`, + but `pyproject` declares just `platformdirs`; pytest and every runtime dep live in + `requirements.txt`, which CI never installed. Fixed in PR #13. + +**Consequence worth carrying forward:** v2.0 through v4.2 were all archived under a +definition of done that included "CI-green." That was never true, and the suite had only +ever run on one Windows machine. It does pass identically on Linux, so no code defect +follows — but the claim was unearned, and the same audit trail should not be trusted for +other "verified" assertions without a spot check. ## Infrastructure State -- Branch master, ahead of origin by e159f24 + this WIP commit. Not pushed. -- Bash tool is bash (use /dev/null, not $null). Commit via `gsd-sdk query commit`. No Claude co-author trailer (user CLAUDE.md). -- Test baseline (v2.0): 354 passed, 2 skipped — run pytest from repo root. + +- **Dashboard server** may still be running in the background on `http://127.0.0.1:8001` + (started via `shoppybot web`, auto-incremented off 8000). Kill it when done. +- **Port 8000** belongs to an unrelated project (`E:\repos\MMFManager` uvicorn backend). Do + not evict it; `shoppybot web` now auto-increments. +- **Seeded test data** is in the real runtime DB at `C:\Users\brand\AppData\Local\shoppybot\`: + items "Seed Amazon Item" / "Seed BestBuy Item", one confirmed order + `112-3456789-0000001`, 24 Amazon price points, and ~612 seeded log lines in today's log. + **This is fixture data, not real activity.** Clear it before any production use. +- **Worktree** at `E:\repos\ShopPyBot-wt-sig` (branch `fix/signal-handler-main-thread`). + `git worktree remove` it once PR #12 merges. `E:\repos\ShopPyBot-wt-cifix` can go now — + PR #13 is merged. -Everything is healthy except the one synthesizer error. The research is strong and converges on a clean 12→16 build order. Resume is mechanical: regenerate SUMMARY → requirements scoping (per-area multiselect) → roadmapper (start phase 12) → approve/commit. +The session started as a routine "check UAT status" and the tool said there was nothing to +do. That was wrong, and chasing why produced everything else. The pattern worth repeating: +when an automated all-clear disagrees with a documented backlog, verify the tool's scope +before believing it. + +Three of the defects found were only reachable by actually running things — the dashboard in +a real browser, the workflow on a real runner. Static review and a green local test suite had +been consistent with all three being fine for months. -Start with: re-spawn `gsd-research-synthesizer` (model sonnet) to overwrite `.planning/research/SUMMARY.md` from STACK/FEATURES/ARCHITECTURE/PITFALLS, verify its content is v3.0 (mentions proxy/CAPTCHA/price), then continue new-milestone.md step 9 (Define Requirements). +Ask the operator to run: + ! gh pr merge 12 --repo thezoid/ShopPyBot --squash --delete-branch + +Then offer to push the 4 local commits on `chore/v4.0-milestone-close`, and pull PR #11's +merge-readiness now that branch protection requires satisfiable contexts. diff --git a/.planning/HANDOFF.json b/.planning/HANDOFF.json index 314e697..3ca35e3 100644 --- a/.planning/HANDOFF.json +++ b/.planning/HANDOFF.json @@ -1,42 +1,54 @@ { "version": "1.0", - "timestamp": "2026-06-07T08:13:27.961Z", - "context": "research", - "milestone": "v3.0", - "milestone_name": "Resilience + Ecosystem", - "workflow": "new-milestone", - "stage": "research-synthesis-incomplete", + "timestamp": "2026-08-01T23:44:34.447Z", + "context": "default", + "milestone": "v4.2", + "milestone_name": "Release Readiness (closed) — post-milestone UAT audit session", + "workflow": "gsd-audit-uat", + "stage": "uat-audit-complete-pending-merges", "status": "paused", "completed_steps": [ - {"step": "gather goals + version", "detail": "v3.0, all four areas FULL (anti-detection hardening, plugin ecosystem, price monitoring, stability/polish)"}, - {"step": "PROJECT.md updated", "detail": "Current Milestone v3.0 section + Active reqs + footer", "commit": "e159f24"}, - {"step": "STATE.md switched", "detail": "state.milestone-switch v3.0; status=planning", "commit": "e159f24"}, - {"step": "research spawned", "detail": "4 gsd-project-researcher agents (Stack/Features/Architecture/Pitfalls) completed; files written but UNCOMMITTED until this WIP commit"} + {"step": "Cross-phase UAT audit", "detail": "80 outstanding items across 66 files in the archived milestone dirs; SDK had falsely reported zero"}, + {"step": "Live browser UAT sweep", "detail": "13 PASS / 1 FAIL / 5 BLOCKED / 2 PARTIAL, driven via Claude-in-Chrome against a seeded local dashboard"}, + {"step": "SEED-003 planted", "detail": "remote plugin manager + third-party liability disclaimer + extensibility framework", "commit": "d485710"}, + {"step": "Port auto-select for `shoppybot web`", "detail": "probe/increment/report + opt-in --open; 18 tests", "commit": "0cebc9e"}, + {"step": "UAT verdicts recorded", "detail": "11 phase artifacts, +246/-40, all frontmatter validated", "commit": "70f31c5"}, + {"step": "STATE.md session record", "detail": "verdict counts, 3 defects, operator actions, outstanding items, doc drift", "commit": "f883f13"}, + {"step": "PR #13 CI fix MERGED", "detail": "runner-context compile fix + requirements.txt install + httpx; CI green for the first time ever", "commit": "4123059"}, + {"step": "PR #12 signal-handler fix", "detail": "rebased onto master, all 4 checks green, CLEAN and awaiting operator merge", "commit": "3f27a2d"}, + {"step": "Operator actions", "detail": "PVR enabled, Actions allowlist widened, CodeQL re-enabled, wiki registry published, RH-07 signed off, branch protection corrected"} ], "remaining_steps": [ - {"step": "synthesize SUMMARY.md", "status": "not_done", "detail": "gsd-research-synthesizer errored (tool result missing) then was stopped. .planning/research/SUMMARY.md on disk is STALE v1 content (Selenium->nodriver) — must regenerate for v3.0."}, - {"step": "define requirements", "status": "not_started", "detail": "scope per area via AskUserQuestion multiselect; new REQ-IDs continuing existing (ANTI-04+, new PRICE-/ECO-/STAB- categories); write .planning/REQUIREMENTS.md (does not exist yet)"}, - {"step": "create roadmap", "status": "not_started", "detail": "spawn gsd-roadmapper, START PHASE NUMBERING AT 12 (continue from 11). Researchers converged on build order: 12 anti-detect infra -> 13 price (parallelizable) -> 14 ecosystem/metadata -> 15 BotService+front-ends -> 16 stability"}, - {"step": "link pending todos", "status": "not_started"} + {"step": "Merge PR #12", "detail": "operator command; the classifier blocks agent-initiated merges"}, + {"step": "Push 4 local commits on chore/v4.0-milestone-close", "detail": "d485710, 0cebc9e, 70f31c5, f883f13"}, + {"step": "Decide on PR #11", "detail": "264 commits; unblocks the Dependabot drain, release-please, gitleaks and modern CodeQL on master"}, + {"step": "Re-test 27-HV-2 / 28-HV-1 / 28-HV-2 running-half", "detail": "all three were blocked by the Start Bot defect; re-run once PR #12 lands"}, + {"step": "10-HV-3 / MC-4", "detail": "needs a `shoppybot web --host 0.0.0.0` restart to verify the banner in both themes"}, + {"step": "29-HV-5", "detail": "needs a DevTools source breakpoint; the doc's stated recipe cannot work"}, + {"step": "MC-1 / MC-2", "detail": "Windows TTY keyring survival + masked prompt; the only two items with no prerequisites"}, + {"step": "Repair 12 drifted UAT docs", "detail": "three of them produce FALSE PASSES as written; see the drift list in STATE.md"}, + {"step": "Fix docs/PLUGIN_REGISTRY.md example row", "detail": "says difficulty 'hard'; AmazonPlugin inherits 'medium' from core/plugin_base.py:78-80"} ], "blockers": [ - {"description": "v3.0 research SUMMARY.md was never generated; the file on disk is the OLD v1 summary. Do NOT consume it as v3.0 research.", "type": "technical", "workaround": "Re-spawn gsd-research-synthesizer (sonnet) reading the 4 fresh v3.0 files to overwrite SUMMARY.md."} + {"description": "Agent cannot merge PRs or write branch protection", "type": "external", "workaround": "Operator runs it with the `!` prefix; worked cleanly for branch protection and PR #13"}, + {"description": "31-HV-3 Dependabot queue will not drain", "type": "external", "workaround": "Two causes: remediating bumps live on chore/v4.0-milestone-close (needs PR #11), AND Dependabot is repo-level PAUSED"}, + {"description": "PR #11 mergeStateStatus reads UNKNOWN", "type": "technical", "workaround": "Re-query after PR #12 lands; it read BLOCKED before branch protection was corrected"} ], - "human_actions_pending": [], - "decisions": [ - {"decision": "v3.0 = all four areas, FULL scope (major release)", "rationale": "user selected all four + full framing", "phase": "v3.0"}, - {"decision": "research-first before requirements", "rationale": "new domains (proxy/CAPTCHA/fingerprint) move fast", "phase": "v3.0"}, - {"decision": "Only one new dep: 2captcha-python 2.0.7 (AsyncTwoCaptcha, MIT)", "rationale": "STACK research; proxy/fingerprint/price/wiki need zero new deps", "phase": "v3.0"}, - {"decision": "Walmart/Target use behavioral managers (HUMAN/Akamai) NOT CAPTCHA-solvable -- keep manual-pause", "rationale": "FEATURES+PITFALLS research", "phase": "v3.0"}, - {"decision": "core/browser_factory.py is the shared hook for proxy/fingerprint/CAPTCHA; ABC setup() default calls it; per-plugin migration, zero forced rewrites of the 7 existing plugins", "rationale": "ARCHITECTURE research", "phase": "v3.0"}, - {"decision": "CAPTCHA/proxy secrets via CredentialStore SECRET_KEYS (TWOCAPTCHA_API_KEY, CAPSOLVER_API_KEY, PROXY_USERNAME, PROXY_PASSWORD); CAPTCHA opt-in default false like SMS gate; price-drop dedup uses SEPARATE columns from stock dedup", "rationale": "ARCHITECTURE+PITFALLS research", "phase": "v3.0"} + "human_actions_pending": [ + {"action": "gh pr merge 12 --repo thezoid/ShopPyBot --squash --delete-branch", "context": "PR #12 is CLEAN with all 4 checks green; closes the only real product defect the audit found", "blocking": true}, + {"action": "Decide whether to merge PR #11", "context": "264 commits; drains the 7 Dependabot alerts and lands release-please + gitleaks + modern CodeQL on master", "blocking": false}, + {"action": "Unpause Dependabot", "context": "GET /repos/thezoid/ShopPyBot/automated-security-fixes returns {\"enabled\":true,\"paused\":true}; no new PRs open until unpaused", "blocking": false}, + {"action": "Add a LICENSE file if open-sourcing", "context": "still none; repo is already public", "blocking": false}, + {"action": "Consider enabling secret scanning + push protection", "context": "both disabled; free on public repos and would partly cover for gitleaks", "blocking": false} ], - "uncommitted_files": [ - ".planning/research/STACK.md", - ".planning/research/FEATURES.md", - ".planning/research/ARCHITECTURE.md", - ".planning/research/PITFALLS.md" + "decisions": [ + {"decision": "Wiki Plugin Registry published headers-only, zero rows", "rationale": "last-verified means confirmed against a live retail site; no plugin has ever been live-verified (19-UAT-1), so any date would be fabricated", "phase": "15"}, + {"decision": "PVR is the final sole security-report channel; no maintainer email", "rationale": "operator sign-off on 32-HV-1, converting an autonomous default into a real decision", "phase": "32"}, + {"decision": "_register_signals fixed with a main-thread guard, not a widened except", "rationale": "signal disposition is process-level and main-thread-only on both platforms; the threaded path already has BotService.stop() for cooperative cancellation", "phase": "n/a"}, + {"decision": "Branch protection requires test (ubuntu-latest), test (windows-latest), CodeQL", "rationale": "the old five matched nothing any workflow emits. Kept the stable `CodeQL` aggregate over `Analyze (python)`, which PR #11 renames to `Analyze (python, none)` via build-mode", "phase": "n/a"}, + {"decision": "httpx==0.28.1 added to requirements.txt rather than a new extra", "rationale": "pytest already lives there; keeps one install path and fixes the undeclared-dep papercut", "phase": "n/a"} ], - "next_action": "Re-spawn gsd-research-synthesizer (model sonnet) to OVERWRITE .planning/research/SUMMARY.md from the 4 v3.0 research files, then resume new-milestone.md at step 9 (Define Requirements) -> step 10 (roadmap, start phase 12).", - "context_notes": "Mid /gsd:new-milestone for v3.0. Phases dir empty (cleared at switch). REQUIREMENTS.md does not exist yet (removed at v2.0 close; new-milestone recreates it). 4 research files are solid and converge on a clean build order. ONLY broken thing = SUMMARY synthesis. Resume = synthesize -> requirements -> roadmap(start 12). Bash tool is bash. Commit via gsd-sdk query commit. No Claude co-author trailer per user CLAUDE.md." + "uncommitted_files": [], + "next_action": "Ask the operator to run `gh pr merge 12 --repo thezoid/ShopPyBot --squash --delete-branch`, then offer to push the 4 local commits and pull PR #11 merge-readiness.", + "context_notes": "Session began as a routine UAT status check and became a real audit. `gsd-sdk query audit-uat` reported total_items: 0 — a FALSE all-clear, because it scans .planning/phases/ which is empty once milestones are archived; 80 real items sat in .planning/milestones/*-phases/. Three defects fell out, none previously caught by any test or milestone review: (1) dashboard Start Bot never worked, _register_signals raising ValueError off the main thread; (2) CI never compiled, ${{ runner.temp }} in job-level env:, 81 runs 81 failures zero jobs; (3) CI never installed dependencies, pyproject declares only platformdirs. CI is now green for the first time in project history: 755 passed / 2 skipped, identical on Ubuntu and Windows. Carry forward: v2.0 through v4.2 were all archived claiming 'CI-green', which was never true. Environment notes that still hold: use .venv/Scripts/python.exe -m pytest, NOT rtk pytest; STATE.md uses a custom prose-field schema the gsd-sdk state verbs cannot parse (hand-edit); a background dashboard server may still be running on 127.0.0.1:8001 (port 8000 belongs to an unrelated MMFManager backend); the session scratchpad directory was wiped mid-run once, so never leave uncommitted work in a worktree placed there." } diff --git a/.planning/MILESTONES.md b/.planning/MILESTONES.md index a5f414d..655ce75 100644 --- a/.planning/MILESTONES.md +++ b/.planning/MILESTONES.md @@ -1,5 +1,47 @@ # Milestones +## v4.2 Release Readiness (Shipped: 2026-07-03) + +**Phases completed:** 6 phases, 20 plans, 45 tasks + +**Delivered:** ShopPyBot closes out its debt-closure and release-hardening backlog: a place-order double-buy idempotency latch, Amazon WAF auto-solve wiring, and real post-login verification harden the acquisition path; CI now runs a real secret scan, a working CodeQL workflow, and Dependabot with a clean alert queue; release-please is seeded and `pyproject.toml` reconciled to 2.0.0 with a refreshed README and a real security-contact channel; platform delay-config fields are harmonized with a back-compat shim and plugins can self-declare their own config section; the dashboard gained a per-plugin log filter and outcome analytics; and the three outstanding v4.1 audit warnings plus stale v4.0/v4.1 doc frontmatter are resolved. + +**Key accomplishments:** + +- **Breakfix hardening (P30):** Place-order-timeout double-buy latch (`_PossiblyPlaced` marker + guard, HIGH), Amazon WAF auto-solve via the existing 2captcha path with the manual-pause fallback preserved, and real post-login DOM/URL verification (`_verify_login_generic`) rolled out to all 7 plugins. +- **CI & security infrastructure (P31):** gitleaks secret-scan CI job + local tracked-secret guard test, CodeQL workflow repaired (retired Node16 actions bumped to `checkout@v6`/`codeql-action@v4`), `.github/dependabot.yml` added and all 7 open vulnerability alerts remediated (cryptography/pydantic-settings/jinja2 bumped). +- **Release automation & community readiness (P32):** `pyproject.toml` reconciled to 2.0.0, release-please seeded and wired for conventional-commit changelog/tagging, README rewritten for the current 7-platform architecture, SECURITY.md/CODE_OF_CONDUCT.md routed through GitHub Private Vulnerability Reporting (placeholder contact removed). +- **Config refactor (P33):** Platform delay-config fields harmonized to `delay_seconds`/`delay_jitter` with a back-compat shim for legacy configs, and a generic per-platform config extension point (`extra="allow"` + `get_platform_config()`) lets any plugin self-declare its own config section with zero core schema edits. +- **Feature completion (P34):** Every log line now carries a `[plugin]` tag; `/api/logs` and the dashboard gained a live plugin filter; a new outcome-analytics endpoint/view computes success-rate and time-to-checkout from confirmed-order records. +- **Audit-fixes & doc-hygiene (P35):** SSR remove-button now degrades gracefully without JS, the raw `last_heartbeat` float no longer leaks from `get_status()`/SSE, the dead `escHtml()` helper is removed, and v4.0/v4.1 planning-artifact frontmatter is reconciled to match each phase's real passing status. + +**Audit:** `.planning/milestones/v4.2-MILESTONE-AUDIT.md` — 20/20 requirements satisfied, 6/6 phases, 6/6 integration boundaries WIRED, 8/9 E2E flows complete. Full suite 940 passed / 2 skipped. Status `tech_debt` (no blockers): 1 code-level gap (BF-02 marker not yet propagated to the 5 community plugins, pre-declared deferred scope), 5 operator-gated GitHub Settings items (PVR toggle, Actions allowlist x2, LICENSE file, conduct-report channel), 1 pending sign-off (RH-07 channel choice made autonomously), and a Nyquist frontmatter lag across all 6 of this milestone's own phases. + +**Known deferred items at close: 8** (see STATE.md → Deferred Items) — 5 phase `human_needed` verification statuses (live-environment checks structurally impossible in CI) + 1 todo (Amazon WAF live-challenge acceptance, manual-pause fallback in place) + 2 dormant seeds (SEED-001 destructive history rewrite stays operator-gated; SEED-002 fully closes once release-please's first Actions run goes green). + +--- + +## v4.1 Dashboard & Observability (Shipped: 2026-06-30) + +**Phases completed:** 6 phases (25-29 + inserted 29.1), 20 plans + +**Delivered:** The optional FastAPI dashboard is redesigned on a zero-Node vendored design system and surfaces live operational observability over SSE, without breaking the CLI-default, localhost-bound, no-CDN posture. + +**Key accomplishments:** + +- **Design system (P25):** Zero-Node vendored CSS split (`tokens.css` / `components.css` / `dashboard.css`), light/dark theme with FOUC-safe inline `` script, uPlot 1.6.32 vendored (no CDN), and the `loadItems()`/`loadCredentials()` XSS vector replaced with `createElement`/`textContent`; MC-4 non-local banner + CSRF gate preserved. +- **Read-only API (P26):** `GET /api/history`, `GET /api/price-history/{link_b64}`, and a filterable `GET /api/logs` (level/search/n) — every sync DB/log read wrapped in `asyncio.to_thread`; `last_error` scrubbed to `exc.__class__.__name__`; CI assertion guards the read path against credential-pattern leaks. +- **SSE infrastructure (P27):** Single `/api/events` stream over a clean cross-thread bridge where uvicorn's `_poll_loop` is the sole producer (bot daemon never touches `asyncio.Queue`); keepalive comments, disconnect cleanup (no generator leak), `retry: 3000`, and a cursor-based log tail. +- **Observability surfaces (P28):** Per-plugin health cards (status badge, monotonic heartbeat-age color bands, error/items counters, confirmed-orders counter), confirmed-buys table, per-item uPlot price charts with explicit empty-state, color-coded filterable log viewer (Follow/pause-on-scroll, 500-line DOM cap), and a uptime status bar. +- **SSE client wiring (P29):** Replaced the 2s `setInterval` polling with a single feature-detected `EventSource('/api/events')` (named status/log listeners, one-shot backfill), a polling fallback for environments without `EventSource`, and a Live/Reconnecting indicator. +- **Tech-debt cleanup (P29.1, inserted):** Closed 3 audit warnings — uPlot loader relocated to `` (cold-load `ReferenceError` race removed), blanket consecutive-line log drop replaced with a one-shot backfill-boundary dedup, and an SSE idle watchdog + REST polling fallback so a stalled-but-open stream flips to Reconnecting and recovers to Live. + +**Audit:** `.planning/milestones/v4.1-MILESTONE-AUDIT.md` (refresh 2026-06-30) — 16/16 requirements satisfied, 6/6 phases, 6/6 integration boundaries WIRED, 5/5 E2E flows complete. Full suite 807 passed / 2 skipped. Status `tech_debt` (no blockers): 2 low-sev warnings (UI-03 SSR remove-button dead handler; `last_heartbeat` cosmetic field in status payload). + +**Known deferred items at close: 8** (see STATE.md → Deferred Items) — 4 `human_needed` verifications (P27/28/29/29.1, live-browser/socket) + 1 partial HUMAN-UAT (29.1, 3 scenarios), all deferred per the autonomous live-UAT policy; 1 todo (Amazon WAF auto-solve, manual-pause fallback in place); 2 dormant seeds (SEED-001 repo scrub, SEED-002 release-please) — release-milestone items. + +--- + ## v4.0 Win-the-Drop (Shipped: 2026-06-25) **Phases completed:** 7 phases, 29 plans diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md index 09154d3..2603c28 100644 --- a/.planning/PROJECT.md +++ b/.planning/PROJECT.md @@ -16,24 +16,81 @@ Target user: technically capable individuals who want automated stock monitoring ## Current State -**Shipped v4.0 Win-the-Drop — Acquisition Core + Reliability (2026-06-25).** Built on the v3.0 resilience/ecosystem layer and the v2.0 modular core (`BotService` API behind a CLI-default front-end plus an optional FastAPI web UI; runtime-selected `CredentialStore` with no plaintext on disk). v4.0 makes the bot complete *verified* orders on limited-release drops and survive multi-hour unattended runs: a central monitor-only gate + `place_order_guarded()` ABC closing the 6-of-7 `test_mode` hole; order-confirmation detection (`purchased` only on a real order number); checkout profile + BestBuy/Amazon form-fill with CVV-at-runtime; one unified `RetryPolicy` with per-step timeouts and idempotent cart-retry; per-coroutine supervisor with browser relaunch, DB read isolation, per-item timeout, and a SIGTERM/SIGINT teardown bridge; Fernet-encrypted session persistence; and a per-plugin health surface (`get_status`, `health_degraded` alert, `shoppybot status`) plus a headless pygame import-crash guard. +**Shipped v4.2 Release Readiness (2026-07-03).** A debt-closure + release-hardening milestone: an unattended run can no longer double-buy on a place-order-stage timeout (write-ahead DB marker + `_PossiblyPlaced` guard, Amazon + BestBuy), Amazon WAF challenges are attempted via the existing 2captcha solver before falling back to manual pause, and all 7 plugins now verify login via real post-login DOM/URL signals instead of assuming success from a click. CI runs a real gitleaks secret scan and a repaired CodeQL workflow, `.github/dependabot.yml` is in place with all 7 open vulnerability alerts remediated, `pyproject.toml` is reconciled to `2.0.0`, and release-please is seeded and wired for conventional-commit changelog/tagging. README is rewritten for the current 7-platform architecture and SECURITY.md/CODE_OF_CONDUCT.md route through GitHub Private Vulnerability Reporting. Platform delay-config fields are harmonized (`delay_seconds`/`delay_jitter`) with a back-compat shim, and a plugin can now self-declare its own config section with zero core schema edits. The dashboard gained a per-plugin log filter and outcome analytics (success-rate, time-to-checkout). Three outstanding v4.1 audit warnings (SSR remove-button, `last_heartbeat` leak, dead `escHtml()`) and stale v4.0/v4.1 doc frontmatter are resolved. -v4.0: 7 phases (18-24) / 29 plans, all complete. Full suite: 755 passed, 2 skipped. Audit status `tech_debt` (no blockers; pre-accepted live-UAT debt). +v4.2: 6 phases (30-35) / 20 plans / 45 tasks, all complete. Full suite: 940 passed, 2 skipped. Audit status `tech_debt` (no blockers): one code-level gap (BF-02 marker not yet propagated to the 5 community plugins) plus a set of operator-gated GitHub Settings actions outstanding — enable Private Vulnerability Reporting, widen the Actions allowlist for `gitleaks`/`release-please`, merge release-please PR #11 to master, and add a LICENSE file if open-sourcing. -**Deferred (carried):** all live-environment UAT (monitor-only/confirmation/form-fill/relaunch/SIGTERM/session/headless) tracked in STATE.md → Deferred Items as the operator's pre-production live-buy checklist; Amazon WAF CAPTCHA auto-solve (manual-pause fallback); public-release hardening — git-history scrub/squash (SEED-001) + release-please tagging (SEED-002). +**Built on v4.1 Dashboard & Observability (2026-06-30):** zero-Node vendored dashboard redesign (3-file CSS split, light/dark, FOUC-safe) surfacing live operational observability over a single `/api/events` SSE stream (health cards, confirmed-buys table, price-history charts, filterable log viewer, uptime bar), backed by read-only REST endpoints with credential-leak CI guards — over the v4.0 Win-the-Drop acquisition/reliability core and the v2.0 modular `BotService`. -**Key constraints (held):** secrets never in config.yml/logs/SQLite plaintext; full card number / CVV never persisted to disk or logs (retailer-saved payment + CVV-at-runtime only); GUI optional, CLI default; the credential-managing web UI binds to localhost by default. +**Deferred (carried):** all live-environment UAT (v4.0 acquisition checks, v4.1 dashboard live-browser/socket checks, v4.2 WAF/double-buy/CI-Actions live checks) tracked in STATE.md → Deferred Items as the operator's pre-production + release checklist; the 5-community-plugin BF-02 marker propagation gap; the destructive half of SEED-001 (public-repo history scrub/squash) stays operator-gated. -## Next Milestone +**Correction recorded 2026-08-02:** everything described above as "shipped" for v4.1 and v4.2 is shipped *on a branch*, not on `master`. A full-repo sweep found the default branch 263 commits behind, PR #11 hard-blocked because its `ci.yml` fails to compile (so the v4.1+v4.2 suite has never run in CI), the built wheel missing every data file (`shoppybot web` cannot start from an installed wheel), and no LICENSE on a public repo. Milestone v5.0 exists to make the mainline and the published artifact match these claims. -**No active milestone.** v4.0 Win-the-Drop shipped 2026-06-25. Start the next cycle with `/gsd:new-milestone` (questioning → research → requirements → roadmap). Phase numbering continues from 24. +**Key constraints (held):** secrets never in config.yml/logs/SQLite plaintext; full card number / CVV never persisted to disk or logs (retailer-saved payment + CVV-at-runtime only); GUI optional, CLI default; the credential-managing web UI binds to localhost by default; zero-Node (no package.json/CDN/external fonts — vendored CSS/JS only); observability is read-only over `get_status()` + DB with no new secrets. -Candidate directions from the v4.0 deferral list: -- **Public-release hardening** — git-history scrub/squash (SEED-001) + release-please version tagging (SEED-002); a dedicated release milestone. -- **Checkout form-fill for the remaining 5 retailers** (v4.0 covers BestBuy + Amazon). +
+Shipped: v4.2 Release Readiness — 2026-07-03 + +**Goal:** Close every outstanding code-actionable item — seeds, breakfixes, audit warnings, deferred sub-features, and release-hardening gaps — so the public repo reaches a stable, release-ready state. + +**Delivered features:** +- Breakfix hardening (P30): place-order-timeout double-buy latch (HIGH), Amazon WAF auto-solve via 2captcha with manual-pause fallback preserved, real post-login DOM/URL verification (`_verify_login_generic`) across all 7 plugins. +- CI & security infrastructure (P31): gitleaks secret-scan CI job + local guard test, repaired CodeQL workflow (checkout@v6/codeql-action@v4), `.github/dependabot.yml` + all 7 open vulnerability alerts remediated. +- Release automation & community readiness (P32): `pyproject.toml` reconciled to 2.0.0, release-please seeded (manifest-mode, python release-type), README rewritten, SECURITY.md/CODE_OF_CONDUCT.md routed through GitHub Private Vulnerability Reporting. +- Config refactor (P33): canonical `delay_seconds`/`delay_jitter` fields with legacy back-compat shim; generic per-platform config extension point (`PlatformsConfig(extra="allow")` + `RetailerPlugin.get_platform_config()`). +- Feature completion (P34): `[plugin]` log tag on every log line + `/api/logs` plugin filter (completes OBS-08); outcome analytics (success-rate, time-to-checkout) over confirmed-order records. +- Audit-fixes & doc-hygiene (P35): SSR remove-button graceful degradation, `last_heartbeat` leak scrubbed from `get_status()`/SSE, dead `escHtml()` removed, v4.0/v4.1 planning-artifact frontmatter reconciled. + +**Constraints held:** CLI default; web optional + localhost bind + CSRF + non-local warning; zero-Node; observability read-only over `BotService.get_status()` + DB, no new secrets; no plaintext secrets, no full card/CVV persistence. + +
+ +
+Shipped: v4.1 Dashboard & Observability — 2026-06-30 + +**Goal:** Redesign the optional FastAPI web dashboard with a polished zero-dependency design system and surface rich live operational observability over SSE, without breaking the CLI-default, localhost-bound, no-Node posture. + +**Delivered features:** +- Vendored zero-Node design system (3-file CSS split: tokens/components/dashboard; light/dark with FOUC-safe inline theming; uPlot 1.6.32 vendored, no CDN/fonts; `loadItems()`/`loadCredentials()` XSS vector fixed). +- Read-only observability REST endpoints (`/api/history`, `/api/price-history/{link_b64}`, filtered `/api/logs`), all `asyncio.to_thread`-wrapped; `last_error` scrubbed; credential-leak CI guard. +- SSE infrastructure: single `/api/events` stream, uvicorn `_poll_loop` sole-producer cross-thread bridge, keepalive, clean disconnect, cursor log tail. +- Four observability surfaces: per-plugin health cards, confirmed-buys table, per-item price-history charts (empty-state), filterable color-coded log viewer (follow + 500-line cap), uptime bar. +- SSE client wiring: `EventSource('/api/events')` replaces the 2s poll, named listeners, polling fallback, Live/Reconnecting indicator; inserted 29.1 cleanup closed 3 audit warnings (uPlot load order, log-dedup, SSE stall watchdog + REST fallback). + +**Constraints held:** CLI default; web optional + localhost bind + CSRF + non-local warning; zero-Node (no package.json/CDN/external fonts); observability read-only over `BotService.get_status()` + DB, no new secrets. + +
+ +## Current Milestone: v5.0 Real Release & Plugin Ecosystem + +**Goal:** Make the default branch, the published artifact, and the public repo actually be what four shipped milestones already claim, then open the plugin framework to third parties with a trust model that survives the fact that importing a plugin is executing it. + +**Premise (established by the 2026-08-02 sweep, 221 evidenced findings):** `master` is 263 commits behind. Every v4.1 and v4.2 artifact — the dashboard, gitleaks, release-please — exists only on the unmerged `chore/v4.0-milestone-close` branch. The default branch is still v4.0, PR #11's test suite has never run in CI (its branch `ci.yml` references `${{ runner.temp }}` in a job-level `env:` and fails to compile), and the built wheel contains zero data files, so `shoppybot web` cannot start on any non-editable install. The claims are ahead of the reality; this milestone closes that gap. + +**Target features:** +- **A. Mainline reconciliation** — fix the branch `ci.yml` compile bug, resolve PR #11 without dropping `httpx`, triage the 4 local commits absent from the PR, land #11 → #12 → Dependabot PRs in dependency order, close the stale #8. +- **B. Distributable artifact** — package-data and a truthful dependency declaration in `pyproject.toml`; installing the wheel must launch `shoppybot web` and play sounds. +- **C. Public-repo readiness** — LICENSE, delete `_deprecated/`, README rewritten for nodriver (not Selenium), `sample.config.yml` rebuilt to include `monitor_only` and every current section, CODEOWNERS, documentation drift. +- **D. Live defect closure** — `/api/bot/start` fire-and-forget reporting, stdin listener EOF spin / None-stdin / executor occupancy, `/api/config` applying to the running process, Discord empty-`url` embeds on `plugin_parked` + `health_degraded`, browser preflight diagnosis, `/api/history` one-shot staleness. +- **E. Scanning to zero** — 7 open Dependabot alerts, 5 real CodeQL alerts, `ci.yml` permissions block, remove the disabled CodeQL workflow, gitleaks as a required check, branch-protection hardening. +- **F. Quality floor** — linter, formatter, and typechecker adopted and wired into CI (none exist today, so CLAUDE.md's own standards are unenforced), coverage measurement, bare-except sites, dead code. +- **G. Community plugin parity** — BF-02 place-order marker on the 5 community plugins, per-step timeouts, `monitor_only` entry guard, confirmation-tab capture. +- **H. Plugin ecosystem (SEED-003)** — user-writable plugin directory, install provenance, machine-readable registry, `PLUGIN_API_VERSION` enforcement, install-time consent gate, capability limits, third-party liability disclaimer, and `install`/`update`/`remove` CLI. +- **I. Ops hardening** — `price_history` index and retention policy. +- **J. UAT repair and triage** — fix the 2 physically impossible test recipes, re-run the 8 items PR #11/#12 unblock, and triage the 61 live-environment items against a stated acceptance bar. + +**Seeds in scope:** SEED-002 (release-please has never executed — register it on master and cut a real release), SEED-001 (retire the destructive history rewrite as a recorded decision: gitleaks across 964 commits found only a test-fixture false positive; keep the LICENSE and public-launch half), SEED-003 (full plugin ecosystem, 16 verified gaps). + +**Sequencing constraints:** A gates E. B gates SEED-002 being worth running. H is the only workstream needing a genuine design pass. Per RETROSPECTIVE.md lesson 4, G and H both warrant a post-verification REVIEW.md deep-review pass — G touches a safety-critical guard, H adds a new unauthenticated input surface. + +## Future Candidate Directions + +Candidates for milestones after v5.0: +- **Checkout form-fill for the remaining 5 retailers** (v4.0 covers BestBuy + Amazon; v5.0 workstream G brings them to safety parity, not checkout parity). +- **Order-confirmation detection for the 5 community plugins** — currently Amazon/BestBuy only; every community plugin falls through the detector. - **Request/API-mode (hybrid) checkout** — faster than DOM but per-site reverse-engineering and an arms race. -- **Outcome analytics** (success rate, time-to-checkout) built on the BUY-04 order records. -- **Richer health/observability on the web dashboard** (beyond the CLI/status payload). +- **Live-environment UAT execution** — v5.0 workstream J triages and repairs the checklist; actually running the ~61 live-retail/live-host items remains operator work gated on a funded 2captcha balance, an Ubuntu host, and a real drop. +- **Process isolation for third-party plugins** — the real fix for the SEED-003 blast-radius problem, and by far the most expensive; v5.0 ships consent plus capability limits instead.
Shipped: v4.0 Win-the-Drop (Acquisition Core + Reliability) — 2026-06-25 @@ -103,17 +160,46 @@ Candidate directions from the v4.0 deferral list: - ✓ Reliability: unified RetryPolicy (one backoff source) — v4.0 (Phase 21) - ✓ Server-safety: headless pygame import-crash guard + SIGTERM/SIGINT teardown bridge — v4.0 (Phases 24, 22) -### Active (next milestone) - -_None yet — run `/gsd:new-milestone` to scope the next cycle. See "Next Milestone" above for candidate directions._ +### Validated (shipped v4.1 Dashboard & Observability) + +- ✓ Redesigned dashboard on a vendored zero-Node design system (tokens/components, light/dark, FOUC-safe) — v4.1 (Phase 25) +- ✓ Read-only observability REST endpoints (history, price-history, filtered logs), to_thread-wrapped + credential-scrubbed — v4.1 (Phase 26) +- ✓ SSE infrastructure: single /api/events stream, uvicorn sole-producer bridge, keepalive, disconnect cleanup — v4.1 (Phase 27) +- ✓ Live observability surfaces: per-plugin health cards, confirmed-buys table, price-history charts, filterable log viewer, uptime bar — v4.1 (Phase 28) +- ✓ SSE client wiring: EventSource replaces polling, fallback, Live/Reconnecting indicator — v4.1 (Phases 29, 29.1) + +### Validated (shipped v4.2 Release Readiness) + +- ✓ Place-order-timeout double-buy idempotency guard (write-ahead DB marker + `_PossiblyPlaced` sentinel), Amazon + BestBuy — v4.2 (Phase 30, BF-02, HIGH) +- ✓ Amazon WAF auto-solve wired to the existing 2captcha solver, manual-pause fallback preserved — v4.2 (Phase 30, BF-01) +- ✓ Plugin login verified via real post-login DOM/URL signals across all 7 plugins — v4.2 (Phase 30, BF-03) +- ✓ Non-destructive secret-scan audit (gitleaks CI job + local guard) — v4.2 (Phase 31, RH-01) +- ✓ CodeQL workflow repaired (retired Node16 actions bumped) — v4.2 (Phase 31, RH-04) +- ✓ `.github/dependabot.yml` + all open dependency vulnerability alerts remediated — v4.2 (Phase 31, RH-05) +- ✓ `pyproject.toml` version reconciled to `2.0.0` + release-please seeded (python release-type) — v4.2 (Phase 32, RH-02, RH-03) +- ✓ README rewritten for the current 7-platform architecture, install, and badges — v4.2 (Phase 32, RH-06) +- ✓ Real maintainer security contact via GitHub Private Vulnerability Reporting, placeholder removed — v4.2 (Phase 32, RH-07) +- ✓ Platform delay-config fields harmonized (`delay_seconds`/`delay_jitter`) with legacy back-compat shim — v4.2 (Phase 33, CFG-01) +- ✓ Generic per-platform config extension point (`extra="allow"` + `get_platform_config()`), zero core schema edits — v4.2 (Phase 33, CFG-02) +- ✓ `[plugin]` log tag on every line + `/api/logs` plugin filter (completes OBS-08) — v4.2 (Phase 34, FC-01) +- ✓ Outcome analytics (success-rate, time-to-checkout) over confirmed-order records — v4.2 (Phase 34, FC-02) +- ✓ SSR items-table remove button works without JS — v4.2 (Phase 35, AF-01) +- ✓ Raw `last_heartbeat` scrubbed from `get_status()`/SSE — v4.2 (Phase 35, AF-02) +- ✓ Dead `escHtml()` helper removed — v4.2 (Phase 35, AF-03) +- ✓ v4.0/v4.1 planning-artifact frontmatter reconciled to match passing validation status — v4.2 (Phase 35, DH-01/02/03) + +### Active + +Milestone v5.0 Real Release & Plugin Ecosystem — workstreams A-J (see Current Milestone above). REQ-IDs assigned in `.planning/REQUIREMENTS.md`. ### Deferred - Request/API-mode (hybrid) checkout (XL arms-race) — follow-on after v4.0 - Virtual-waiting-room / queue survival: Queue-it, PerimeterX, Akamai, DataDome (XL) — follow-on - Multi-account / multi-profile parallel attempts (XL, most ToS-hostile) — follow-on, opt-in if ever -- Amazon WAF CAPTCHA auto-solve — re-deferred (todo: `waf-auto-solve-followup`) -- Public-release hardening: git-history scrub/squash (SEED-001) + release-please tagging (SEED-002) — when a release milestone is scoped +- Amazon WAF CAPTCHA live-challenge acceptance — code wiring shipped v4.2 (BF-01); live-challenge proof against a real AWS-WAF challenge stays operator debt +- BF-02 marker propagation to the 5 community plugins (Walmart, Target, GameStop, NewEgg, SquareEnix) — mechanical follow-up, closes the residual double-buy exposure +- Destructive public-repo history scrub/squash (SEED-001 remainder) — operator-gated one-time action, when ready to make the repo public. Non-destructive audit shipped v4.2 (RH-01); SEED-002 release-please tagging is code-complete (v4.2), pending first live Actions run ### Out of Scope @@ -141,6 +227,18 @@ _None yet — run `/gsd:new-milestone` to scope the next cycle. See "Next Milest | (v4.0) One unified `RetryPolicy` | Supervisor-restart and cart-retry share one backoff module so the two retry concepts cannot diverge or compound into a runaway loop | ✓ Good | | (v4.0) Retailer-saved payment + CVV-at-runtime | Never persist full card/PAN (PCI scope); CVV via `getpass`, never logged; AST CI assertion guards against leaks | ✓ Good | | (v4.0) Live-environment UAT deferred as tracked debt | Live retail checkout is ToS/legal risk in CI; confirmation/form-fill selectors verified by manual UAT, tracked in STATE.md Deferred Items | — Pending (operator live-buy checklist) | +| (v4.1) uPlot vendored, no CDN/Node | Zero-Node is a hard constraint; uPlot is MIT, ~52KB, Canvas, dependency-free — vendored under web/static | ✓ Good | +| (v4.1) 3-file CSS split (tokens/components/dashboard) | Token-only component rules keep theming maintainable and FOUC-safe; each file under 200 lines | ✓ Good | +| (v4.1) uvicorn _poll_loop is the SOLE SSE producer | Bot daemon thread never touches asyncio.Queue; avoids the cross-loop race that is the highest-risk SSE pitfall | ✓ Good | +| (v4.1) No new Python deps for SSE | Raw starlette StreamingResponse(text/event-stream) covers all needs; no sse-starlette, no FastAPI upgrade | ✓ Good | +| (v4.1) Live-browser/socket UAT deferred as tracked debt | SSE/EventSource and visual rendering need a real browser/live socket; all automated assertions GREEN, live checks tracked in STATE.md | — Pending (operator dashboard checklist) | +| (v4.2) BF-02 write-ahead DB marker + `_PossiblyPlaced` sentinel | Makes the place-order stage non-retryable once clicked, reusing the existing orchestrator alert pattern instead of a new state machine | ✓ Good | +| (v4.2) `_verify_login_generic` shared login-verification mechanism | One verification mechanism for all 7 plugins avoids per-plugin duplication; honest URL/DOM-absence signal even without live-verified selectors | ✓ Good | +| (v4.2) Amazon WAF auto-solve reuses the existing 2captcha solver path | No new CAPTCHA integration; fails safe to the pre-existing manual-pause fallback on any non-success path | ✓ Good | +| (v4.2) `PlatformsConfig(extra="allow")` + `get_platform_config()` extension point | Lets any plugin self-declare its own config section with zero core `config_schema.py` edits; the 7 built-in platforms keep full strict validation | ✓ Good | +| (v4.2) Canonical `delay_seconds`/`delay_jitter` with legacy back-compat shim | Unifies platform delay-config naming without breaking existing configs; activated Amazon/BestBuy poll jitter for the first time | ✓ Good | +| (v4.2) RH-07 resolved to GitHub Private Vulnerability Reporting only, no published email | PVR is GitHub-native and audit-logged; avoids publishing a personal maintainer address | — Pending (operator sign-off; decision made autonomously in operator's absence) | +| (v4.2) BF-02 marker propagation scoped to Amazon + BestBuy only this milestone | The 5 community plugins are independently EXPERIMENTAL/selector-unverified, lowering real-world exposure; deliberate scope cut, not a miss | ⚠️ Revisit (residual double-buy risk identical in mechanism to the closed bug) | ## Evolution @@ -160,4 +258,4 @@ This document evolves at phase transitions and milestone boundaries. 4. Update Context with current state --- -*Last updated: 2026-06-25 — after v4.0 Win-the-Drop milestone (shipped)* +*Last updated: 2026-08-02 — milestone v5.0 Real Release & Plugin Ecosystem scoped from a 221-finding full-repo sweep (PRs, security/quality scans, outstanding UAT, seed gaps, defect re-verification) plus an adversarial completeness pass* diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md new file mode 100644 index 0000000..1fdd22d --- /dev/null +++ b/.planning/REQUIREMENTS.md @@ -0,0 +1,327 @@ +# Requirements: ShopPyBot v5.0 Real Release & Plugin Ecosystem + +**Defined:** 2026-08-02 +**Core Value:** A drop-in plugin framework that lets the community add new retail platform integrations by placing a single Python file in `plugins/`, with no core changes required. + +**Basis:** A 221-finding full-repo sweep (open PRs, security and quality scans, outstanding UAT debt, seed gaps, defect re-verification) plus an adversarial completeness pass, plus four researchers and a synthesis pass scoped to workstream H. Every requirement below traces to evidence in that sweep or in `.planning/research/`. + +## v1 Requirements + +### A. Mainline Reconciliation (MAIN) + +`master` is 263 commits behind. Every v4.1 and v4.2 artifact exists only on `chore/v4.0-milestone-close`. Nothing else in this milestone is durable until the default branch is real. + +- [ ] **MAIN-01**: The branch's `ci.yml` compiles, so the v4.1+v4.2 test suite runs in CI for the first time (`SHOPBOT_DATA_DIR: ${{ runner.temp }}/shopbot` currently sits in job-level `env:` where the `runner` context does not exist, producing 0-job failures) +- [ ] **MAIN-02**: PR #11's two conflicts (`.github/dependabot.yml` add/add, `requirements.txt` content) are resolved without dropping `httpx==0.28.1` +- [ ] **MAIN-03**: The 4 local commits present on the working branch but absent from PR #11's head are triaged and deliberately included or excluded, not swept in as a side effect of conflict resolution +- [ ] **MAIN-04**: PR #11 is merged, so `gitleaks.yml`, `release-please.yml`, and every v4.1/v4.2 artifact exist on the default branch +- [x] **MAIN-05**: PR #12 (signal handlers registered off the main thread) is merged +- [x] **MAIN-06**: Stale PR #8 is closed rather than merged (open since 2023, conflicting, superseded because master already carries urllib3 2.7.0) +- [ ] **MAIN-07**: Dependabot PRs #15 through #20 are merged in a conflict-safe order, accounting for #19 and #20 colliding in `ci.yml` + +### B. Distributable Artifact (PKG) + +The built wheel does not run. This is the precondition for release-please being worth anything. + +- [ ] **PKG-01**: An installed wheel contains `web/static/*`, `web/templates/*`, and `sounds/*`, so `create_app()` does not raise `RuntimeError` on the `StaticFiles` mount +- [ ] **PKG-02**: `pyproject.toml` declares every actual runtime dependency, not just `platformdirs==4.10.0` +- [ ] **PKG-03**: `websockets`, `starlette`, `httpx`, and `requests` are each declared in the correct place (all four are imported or required today and none is declared where it is used) +- [ ] **PKG-04**: Dead `selenium` and `webdriver-manager` pins are removed from `requirements.txt`, which also removes a recurring Dependabot noise source +- [ ] **PKG-05**: A CI job installs the built wheel into a clean environment and asserts that `shoppybot web` starts and a sound file resolves +- [ ] **PKG-06**: `bundled_plugins_dir()` is verified to resolve correctly from an installed wheel (blocks EXT-03; if it fails, the fix is `importlib.resources` and belongs to this workstream) + +### C. Public-Repo Readiness (PUB) + +- [ ] **PUB-01**: A LICENSE file exists, so the public repo carries an actual license grant +- [ ] **PUB-02**: `_deprecated/` (roughly 800 lines documenting a retired `settings.json` config format) is removed from the default branch +- [ ] **PUB-03**: README's Prerequisites and Setup sections describe nodriver, not Selenium and `chromedriver` and `selenium.driver_path` +- [ ] **PUB-04**: `sample.config.yml` covers every section `AppConfig` declares, including `debug.monitor_only`, and no longer advertises `available.short_url` or `available.alert_type`, which nothing reads +- [ ] **PUB-05**: A CODEOWNERS file exists so code-owner review can be required +- [ ] **PUB-06**: CLAUDE.md describes the current architecture rather than the retired v1 one +- [ ] **PUB-07**: SECURITY.md no longer carries the stale "Operator note: PVR must be enabled" blockquote (it is enabled) +- [ ] **PUB-08**: Documented drift is corrected: dashboard section count, `python main.py` named as the web launcher, hardcoded banner hex values, cold-load theme default, MC-3's polling description, and `docs/PLUGIN_REGISTRY.md`'s difficulty value for AmazonPlugin +- [ ] **PUB-09**: SEED-001's destructive history rewrite is formally retired as a recorded decision, on the evidence that gitleaks across 964 commits found exactly one test-fixture false positive + +### D. Live Defect Closure (FIX) + +- [ ] **FIX-01**: `POST /api/bot/start` reports a real failure instead of returning 200 unconditionally when the bot loop dies +- [ ] **FIX-02**: The stdin listener thread does not busy-spin at EOF, and does not force-fire every manual-intervention gate in a headless or dashboard-driven run +- [ ] **FIX-03**: A stdin listener failure surfaces rather than dying silently when `sys.stdin` is None +- [ ] **FIX-04**: The blocking stdin readline no longer occupies the default executor and stalls `asyncio.run()` teardown for up to 300 seconds +- [ ] **FIX-05**: `POST /api/config` either applies to the running process or reports that it did not, so `test_mode` cannot silently remain at its old value +- [ ] **FIX-06**: `plugin_parked` and `health_degraded` notifications are actually delivered (both currently build a Discord embed with an empty `url`, which the API rejects, so the two alerts that exist only for unattended operation are the two that never arrive) +- [ ] **FIX-07**: The notification `action` vocabulary is documented for all six values it emits, and a parked plugin is visually distinguishable from a successful purchase +- [ ] **FIX-08**: The plugin name is escaped and length-capped before reaching Discord, email, and SMS bodies (attacker-controlled once EXT ships) +- [ ] **FIX-09**: A host without Chrome produces an actionable message rather than every plugin failing silently while the dashboard reports the bot as started +- [ ] **FIX-10**: The Confirmed Orders view refreshes rather than being fetched exactly once per page load +- [ ] **FIX-11**: `test_post_config_allowlisted_key` patches the correct module path and actually asserts the write it claims to test + +### E. Scanning to Zero (SCAN) + +Depends on A: several of these live on workflows that are not on master yet. + +- [ ] **SCAN-01**: All 7 open Dependabot alerts are closed +- [ ] **SCAN-02**: The two high-severity clear-text logging and storage CodeQL alerts in `logger.py` are resolved +- [ ] **SCAN-03**: The 3 production `py/incomplete-url-substring-sanitization` alerts are resolved +- [ ] **SCAN-04**: The 19 test-file url-sanitization alerts are dismissed or suppressed deliberately, so the queue reflects real findings +- [ ] **SCAN-05**: `ci.yml` carries an explicit `permissions` block (CodeQL alert #30) +- [ ] **SCAN-06**: The `disabled_manually` `codeql-analysis.yml` is removed from master, since default setup replaced it +- [ ] **SCAN-07**: gitleaks is a required status check on master +- [ ] **SCAN-08**: Branch protection on master requires at least one review and applies to admins +- [ ] **SCAN-09**: Secret scanning non-provider patterns and validity checks are enabled +- [ ] **SCAN-10**: The 17-month-stale `origin/dev` branch and its 4 rotted workflows are resolved +- [ ] **SCAN-11**: Third-party actions are pinned by commit SHA rather than floating major tags + +### F. Quality Floor (QUAL) + +No linter, formatter, or typechecker is configured anywhere, so CLAUDE.md's own code standards are entirely unenforced. + +- [ ] **QUAL-01**: A linter and formatter are adopted, configured, and enforced in CI +- [ ] **QUAL-02**: Type checking is adopted and enforced in CI +- [ ] **QUAL-03**: Findings from QUAL-01 and QUAL-02 are fixed, or explicitly baselined with the baseline recorded +- [ ] **QUAL-04**: Test coverage is measured in CI +- [ ] **QUAL-05**: The 11 `except Exception: pass` sites either handle or propagate, with no silent swallowing +- [ ] **QUAL-06**: Dead `amazon_bot.py` and `bestbuy_bot.py` are removed +- [ ] **QUAL-07**: The two tests that skip silently when `config.yml` is absent (one of them a plaintext-secrets security check) fail loudly instead +- [ ] **QUAL-08**: The cross-test mock leak producing a never-awaited coroutine warning is fixed +- [ ] **QUAL-09**: The 9 files over 300 lines and 71 functions over 30 lines are either reduced or explicitly accepted with recorded rationale + +### G. Community Plugin Parity (PAR) + +- [ ] **PAR-01**: All 5 community plugins pass `order_marker_link` to `place_order_guarded`, closing the residual double-buy exposure BF-02 was created to close +- [ ] **PAR-02**: Community plugins have per-step timeouts, so a hung selector cannot block the plugin loop indefinitely +- [ ] **PAR-03**: Community plugins have the `monitor_only` entry guard, implemented as one mechanism with EXT-09 rather than two that can diverge +- [ ] **PAR-04**: Community plugins set `_last_tab` or override `get_active_tab`, so confirmation detection reads the right tab +- [ ] **PAR-05**: Community plugins maintain `_checkout_stage` through the flow, so error logs carry stage context +- [ ] **PAR-06**: Plugin routing is deterministic (`setup_for_items` currently populates `_active_plugins` by iterating a `set`, so with overlapping `domain_patterns` the winner varies by `id()` hash between runs, today, with the 7 bundled plugins) + +### H. Plugin Ecosystem (EXT) + +SEED-003. Trust model is consent plus SHA and content pinning plus honest provenance, explicitly not a sandbox. See `.planning/research/SUMMARY.md` for the reconciled design and the recorded decisions. + +- [ ] **EXT-01**: No plugin can crash bot start, the dashboard render, `plugins list`, or `run_plugin` by omitting an attribute or raising in `__init__` +- [ ] **EXT-02**: `plugins/example_plugin.py` routes its purchase through `place_order_guarded()`, so the canonical template authors copy stops teaching the `test_mode` and BF-02 bypass +- [ ] **EXT-03**: Every plugin path resolves through `core/paths.py`, and a user-writable plugin root exists outside the package +- [ ] **EXT-04**: Bundled plugins win a filename collision, and a third-party plugin claiming a bundled domain is refused at load +- [ ] **EXT-05**: `PLUGIN_API_VERSION` is enforced at load instead of being a decorative constant nothing reads +- [ ] **EXT-06**: Every installed plugin has a provenance record: source, numeric owner ID, resolved commit SHA, content hash, declared API version, and the consent that authorised it +- [ ] **EXT-07**: Plugin file integrity is verified before `exec_module`, not only at install time, because plugin module bodies re-execute on every registry construction including an ordinary dashboard page render +- [ ] **EXT-08**: A run lock lets lifecycle commands tell the truth about whether a bot is running +- [ ] **EXT-09**: Third-party plugins are disarmed for checkout by default, independent of the global `monitor_only` setting, with a separate typed opt-in to arm +- [ ] **EXT-10**: Credential access is scoped per platform, and a third-party plugin never receives the CVV +- [ ] **EXT-11**: `plugins install [@ref]` fetches by resolved commit SHA and runs static pre-flight checks that never import the candidate +- [ ] **EXT-12**: A byte-level lint refuses plugin files containing invisible codepoints, bidi overrides, variation selectors, or Private Use Area characters outside strings and comments +- [ ] **EXT-13**: Install requires typed consent naming the source repo and owner ID, with no bare bypass flag +- [ ] **EXT-14**: `plugins update`, `remove`, `verify`, and `outdated` exist; update re-consents on any change; remove prints a credential rotation checklist +- [ ] **EXT-15**: A third-party plugin disclaimer appears at the point of decision and in SECURITY.md, README, and PLUGIN_DEV.md, and states plainly that third-party installs receive no review and that no revocation mechanism exists +- [ ] **EXT-16**: A machine-readable plugin registry exists as the in-repo source of truth, retiring the hand-maintained wiki table (REG-01, outstanding across two milestones) +- [ ] **EXT-17**: A CI guard fails the build if any shipped artifact describes third-party plugins as sandboxed, isolated, curated, verified, or safe + +### I. Ops Hardening (OPS) + +- [ ] **OPS-01**: `price_history` has an index on `item_link`, so per-chart lookups do not scan an append-only table that grows by roughly 15k rows per day for a 5-item watch +- [ ] **OPS-02**: `price_history` has a retention policy + +### J. UAT Repair and Triage (UAT) + +- [ ] **UAT-01**: 29-HV-5's recipe is rewritten to something physically possible (the `window.EventSource = undefined` plus reload recipe cannot work, because reload restores it) +- [ ] **UAT-02**: 29.1-UAT-2's stall-trigger recipe is corrected to be reliable +- [ ] **UAT-03**: The 8 UAT items that PR #11 and PR #12 unblock are re-run and their verdicts recorded +- [ ] **UAT-04**: The roughly 61 live-environment items are triaged against a stated acceptance bar and consolidated into one operator checklist, rather than being spread across 88 archived artifact files +- [ ] **UAT-05**: The `audit-uat` tooling gap is documented in-repo (it scans `.planning/phases/`, which is empty once milestones are archived, and reported zero outstanding items while 80 sat in `.planning/milestones/`) +- [ ] **UAT-06**: The "CI-green" claim in the v2.0 through v4.2 archives is corrected, since CI had never compiled when those milestones were archived + +## v2 Requirements + +Acknowledged, deferred, not in this roadmap. + +### Plugin Isolation + +- **ISO-01**: Process isolation for third-party plugins. The only real containment boundary; everything v5.0 ships is consent and integrity, not isolation. Windows cost specifically was not scoped. + +### Community Retailer Parity + +- **CRP-01**: Checkout form-fill for Walmart, Target, GameStop, NewEgg, and SquareEnix (v4.0 covers Amazon and BestBuy only) +- **CRP-02**: Order-confirmation detection for the 5 community plugins, which currently always fall through the detector +- **CRP-03**: CVV entry and quantity selection for community plugins +- **CRP-04**: CAPTCHA detection and solve path for community plugins +- **CRP-05**: Resolution of the 25 unverified-selector markers across the community plugins, which is live-retail blocked + +### Reliability + +- **REL-01**: Event-loop stall watchdog. `asyncio.timeout` cannot preempt a blocking plugin, and the exposure exists today with the 7 bundled plugins. Scope during roadmapping; do not let it silently expand EXT. + +**Roadmap disposition (2026-08-02):** REL-01 stays in v2. Research `SUMMARY.md` Gap 5 asked whether the stall watchdog belongs to H, to G, or to a deferred reliability item; it is a reliability control rather than a distribution control, the exposure is identical with the 7 bundled plugins, and folding it into H would silently expand that workstream. Recorded as a decision, not an omission. + +## Out of Scope + +| Feature | Reason | +|---------|--------| +| Process isolation / in-process sandbox | No in-process Python sandbox is a real boundary. PEP 578 says audit hooks are not sandboxing, RestrictedPython disclaims it, pysandbox was abandoned as broken by design. Shipping one would be theatre. Deferred to v2 as ISO-01 | +| Plugin-declared dependencies | `pip --no-deps` still executes a source distribution's `setup.py` during resolution, which is a second and earlier execution channel in a workstream premised on execution being uncontainable. Plugins may import the stdlib plus what ShopPyBot already depends on | +| Hosted plugin marketplace, ratings, stars, install counts, telemetry | Gameable signals are worse than no signal (GlassWorm faked popularity metrics), and this project has deliberately avoided a telemetry surface everywhere else | +| Auto-update, `update --all`, any scheduled path into the update code | Converts every trust-transfer incident from catchable to overnight, on a machine holding a credential store and a live checkout path | +| Hot reload after install, update, or remove | Non-unloadable modules plus live browser subprocesses plus supervised restart; the likely outcome is two plugin instances racing toward the same Place Order click. Restart is required and every mutation says so | +| Maintainer review gate on plugin install | Reintroduces the PR bottleneck SEED-003 exists to bypass | +| Generic Git-host plugin fetch | The point at which a `git` subprocess becomes unavoidable, which is a new environmental precondition on users without a dev toolchain. GitHub-only for v1, deferred rather than silently unsupported | +| Executing the ~61 live-environment UAT items | Requires live retail checkout, a funded 2captcha balance, live proxies, an Ubuntu host, and a real drop. J triages and repairs the checklist; running it is operator work | +| Destructive public-repo history rewrite | SEED-001's remaining half. gitleaks across 964 commits found one test-fixture false positive, so the rewrite is unjustified. Retired as a decision under PUB-09 rather than carried as debt | +| Request/API-mode hybrid checkout | Per-site reverse engineering and an arms race; unchanged from prior milestones | + +## Traceability + +Populated during roadmap creation, 2026-08-02. Every v1 requirement maps to exactly one phase. + +| Requirement | Phase | Status | +|-------------|-------|--------| +| MAIN-01 | Phase 36 | Pending | +| MAIN-02 | Phase 36 | Pending | +| MAIN-03 | Phase 36 | Pending | +| MAIN-04 | Phase 36 | Pending | +| MAIN-05 | Phase 36 | Complete | +| MAIN-06 | Phase 36 | Complete | +| MAIN-07 | Phase 36 | Pending | +| PKG-01 | Phase 37 | Pending | +| PKG-02 | Phase 37 | Pending | +| PKG-03 | Phase 37 | Pending | +| PKG-04 | Phase 37 | Pending | +| PKG-05 | Phase 37 | Pending | +| PKG-06 | Phase 37 | Pending | +| PUB-01 | Phase 40 | Pending | +| PUB-02 | Phase 40 | Pending | +| PUB-03 | Phase 40 | Pending | +| PUB-04 | Phase 40 | Pending | +| PUB-05 | Phase 40 | Pending | +| PUB-06 | Phase 40 | Pending | +| PUB-07 | Phase 40 | Pending | +| PUB-08 | Phase 40 | Pending | +| PUB-09 | Phase 40 | Pending | +| FIX-01 | Phase 41 | Pending | +| FIX-02 | Phase 41 | Pending | +| FIX-03 | Phase 41 | Pending | +| FIX-04 | Phase 41 | Pending | +| FIX-05 | Phase 41 | Pending | +| FIX-06 | Phase 41 | Pending | +| FIX-07 | Phase 41 | Pending | +| FIX-08 | Phase 41 | Pending | +| FIX-09 | Phase 41 | Pending | +| FIX-10 | Phase 41 | Pending | +| FIX-11 | Phase 41 | Pending | +| SCAN-01 | Phase 38 | Pending | +| SCAN-02 | Phase 38 | Pending | +| SCAN-03 | Phase 38 | Pending | +| SCAN-04 | Phase 38 | Pending | +| SCAN-05 | Phase 38 | Pending | +| SCAN-06 | Phase 38 | Pending | +| SCAN-07 | Phase 38 | Pending | +| SCAN-08 | Phase 38 | Pending | +| SCAN-09 | Phase 38 | Pending | +| SCAN-10 | Phase 38 | Pending | +| SCAN-11 | Phase 38 | Pending | +| QUAL-01 | Phase 39 | Pending | +| QUAL-02 | Phase 39 | Pending | +| QUAL-03 | Phase 39 | Pending | +| QUAL-04 | Phase 39 | Pending | +| QUAL-05 | Phase 39 | Pending | +| QUAL-06 | Phase 39 | Pending | +| QUAL-07 | Phase 39 | Pending | +| QUAL-08 | Phase 39 | Pending | +| QUAL-09 | Phase 39 | Pending | +| PAR-01 | Phase 45 | Pending | +| PAR-02 | Phase 45 | Pending | +| PAR-03 | Phase 45 | Pending | +| PAR-04 | Phase 45 | Pending | +| PAR-05 | Phase 45 | Pending | +| PAR-06 | Phase 45 | Pending | +| EXT-01 | Phase 42 | Pending | +| EXT-02 | Phase 42 | Pending | +| EXT-03 | Phase 43 | Pending | +| EXT-04 | Phase 43 | Pending | +| EXT-05 | Phase 43 | Pending | +| EXT-06 | Phase 44 | Pending | +| EXT-07 | Phase 44 | Pending | +| EXT-08 | Phase 44 | Pending | +| EXT-09 | Phase 46 | Pending | +| EXT-10 | Phase 46 | Pending | +| EXT-11 | Phase 47 | Pending | +| EXT-12 | Phase 47 | Pending | +| EXT-13 | Phase 47 | Pending | +| EXT-14 | Phase 48 | Pending | +| EXT-15 | Phase 49 | Pending | +| EXT-16 | Phase 49 | Pending | +| EXT-17 | Phase 49 | Pending | +| OPS-01 | Phase 50 | Pending | +| OPS-02 | Phase 50 | Pending | +| UAT-01 | Phase 50 | Pending | +| UAT-02 | Phase 50 | Pending | +| UAT-03 | Phase 50 | Pending | +| UAT-04 | Phase 50 | Pending | +| UAT-05 | Phase 50 | Pending | +| UAT-06 | Phase 50 | Pending | + +**Coverage:** + +- v1 requirements: 84 total +- Mapped to phases: 84 +- Unmapped: 0 +- Duplicated across phases: 0 + +**Per-phase requirement counts:** + +| Phase | Name | Requirements | Count | +|-------|------|--------------|-------| +| 36 | Mainline Reconciliation | MAIN-01..07 | 7 | +| 37 | Distributable Artifact | PKG-01..06 | 6 | +| 38 | Scanning to Zero | SCAN-01..11 | 11 | +| 39 | Quality Floor | QUAL-01..09 | 9 | +| 40 | Public-Repo Readiness | PUB-01..09 | 9 | +| 41 | Live Defect Closure | FIX-01..11 | 11 | +| 42 | Plugin Registry Hardening | EXT-01, EXT-02 | 2 | +| 43 | Plugin Roots, Precedence & API Version Gate | EXT-03, EXT-04, EXT-05 | 3 | +| 44 | Provenance, Load-Boundary Integrity & Run Lock | EXT-06, EXT-07, EXT-08 | 3 | +| 45 | Community Plugin Parity + Pre-Transfer Arming Gate | PAR-01..06 | 6 | +| 46 | Trust Tiers & Capability Reduction | EXT-09, EXT-10 | 2 | +| 47 | Fetch, Pre-Flight, Install & Consent | EXT-11, EXT-12, EXT-13 | 3 | +| 48 | Plugin Lifecycle — update, remove, verify, outdated | EXT-14 | 1 | +| 49 | Trust Documentation, Registry & Vocabulary Guard | EXT-15, EXT-16, EXT-17 | 3 | +| 50 | Ops Hardening & UAT Consolidation | OPS-01..02, UAT-01..06 | 8 | +| | **Total** | | **84** | + +## Sequencing Constraints + +Hard dependencies the roadmap must respect: + +1. **A before E.** Several SCAN requirements target workflows that do not exist on master until PR #11 lands. +2. **B before SEED-002 is worth running.** release-please exists to publish an artifact that currently cannot start. +3. **PKG-06 before EXT-03.** If the bundled plugin root does not survive a wheel install, the fix is `importlib.resources` and belongs to B. +4. **EXT-01 before every other EXT.** Each later step multiplies the number of non-conforming plugins reaching paths that currently raise. +5. **EXT-09 before EXT-11.** The disarm default must already be true when install ships, so no released state has a stranger's freshly installed plugin able to buy by default. +6. **PAR-03 and EXT-09 are one mechanism.** Both build a pre-transfer gate at the same orchestrator site. +7. **EXT-15 gates milestone completion**, not phase ordering. The disclaimer is the precondition that makes the plugin manager defensible. + +**Roadmap satisfaction (2026-08-02):** + +| # | Satisfied by | +|---|--------------| +| 1 | Phase 36 → Phase 38 | +| 2 | Phase 37 precedes any release cut | +| 3 | Phase 37 (PKG-06) → Phase 43 (EXT-03) | +| 4 | Phase 42 is the first EXT phase | +| 5 | Phase 46 (EXT-09) → Phase 47 (EXT-11) | +| 6 | Phase 45 builds the pre-transfer arming gate for the community plugins; adjacent Phase 46 extends the same gate with the third-party disarm default. One mechanism, built once | +| 7 | Phase 49 carries EXT-15 as a milestone gate; Phase 50 may follow it without violating the constraint | + +## Review Criteria + +Per RETROSPECTIVE.md lesson 4, workstreams G and H both warrant a post-verification REVIEW.md deep pass: G touches a safety-critical guard, H adds a new unauthenticated input surface. That review carries two non-negotiable criteria: + +- For every shipped control, one sentence naming what it does not stop. +- No shipped artifact describes third-party plugins as sandboxed, isolated, curated, verified, or safe. EXT-17 makes this mechanical. + +**Assigned phases (2026-08-02):** + +- **Phase 45** (Community Plugin Parity + Pre-Transfer Arming Gate) — the safety-critical guard. +- **Phase 47** (Fetch, Pre-Flight, Install & Consent) — the new unauthenticated input surface. Phase 48 extends the same surface and re-runs both criteria against `update` and `remove` before closing, under the Phase 47 review's scope. + +*Requirements defined: 2026-08-02* +*Last updated: 2026-08-02 — traceability populated during roadmap creation; 84/84 mapped across Phases 36-50* diff --git a/.planning/RETROSPECTIVE.md b/.planning/RETROSPECTIVE.md index 25792de..57feb4b 100644 --- a/.planning/RETROSPECTIVE.md +++ b/.planning/RETROSPECTIVE.md @@ -44,6 +44,86 @@ --- +## Milestone: v4.1 — Dashboard & Observability + +**Shipped:** 2026-06-30 +**Phases:** 6 (25-29 + inserted 29.1) | **Plans:** 20 | **Suite:** 807 passed, 2 skipped + +### What Was Built +- Zero-Node vendored design system: 3-file CSS split (tokens/components/dashboard), light/dark with FOUC-safe inline `` theming, uPlot 1.6.32 vendored, `loadItems()`/`loadCredentials()` XSS fix (P25). +- Read-only observability REST endpoints (`/api/history`, `/api/price-history/{link_b64}`, filtered `/api/logs`), all `asyncio.to_thread`-wrapped, `last_error` scrubbed, credential-leak CI guard (P26). +- SSE infrastructure: single `/api/events` stream, uvicorn `_poll_loop` sole-producer cross-thread bridge, keepalive, disconnect cleanup, cursor log tail (P27). +- Four observability surfaces: per-plugin health cards, confirmed-buys table, per-item uPlot price charts with empty-state, filterable color-coded log viewer (follow + 500-cap), uptime bar (P28). +- SSE client wiring: `EventSource('/api/events')` replaces the 2s poll, named listeners, polling fallback, Live/Reconnecting indicator (P29); inserted 29.1 closed 3 audit warnings (uPlot load order, log-dedup, SSE stall watchdog + REST fallback). + +### What Worked +- **Spike-first on the highest-risk phase.** P27 (SSE) was flagged highest-risk up front and validated the lifespan + `asyncio.create_task` + `SseHub` wiring before full build. The cross-loop race (the known pitfall) never materialized because the "uvicorn `_poll_loop` is the sole producer; bot thread never touches `asyncio.Queue`" rule was designed in, not retrofitted. +- **RED test scaffold per phase.** Each phase opened with a Wave 0 failing-test scaffold (TestClient static-template assertions for the frontend phases), so GREEN was a concrete target and regressions were guarded. +- **Audit-driven cleanup loop.** The 2026-06-28 audit surfaced 4 integration warnings; rather than ship them as raw debt, an inserted Phase 29.1 closed 3 of them with file:line-verified fixes and 5 new tests, and the milestone-close audit refresh confirmed it. The bot's own audit fed the next unit of work. +- **Zero-Node discipline held.** No package.json, no CDN, no external fonts; charting solved by vendoring a tiny MIT lib and raw `StreamingResponse` covered SSE with no new Python deps. + +### What Was Inefficient +- **Inserted decimal phase had no top-level ROADMAP checkbox.** Phase 29.1 was complete on disk (4/4 plans + summaries) but `roadmap.analyze` reported `roadmap_complete:false` because the parser keys on a `- [x] ... Phase 29.1` list item the inserted section never got. An autonomous run would have tried to re-execute it; this close had to add the checkbox first. +- **SUMMARY `requirements` frontmatter missing on P27/28/29.** Those plan summaries omit the `requirements` field (P28 uses none; P27/29 use `dependency_graph`), so the milestone audit's 3-source cross-reference dropped 6 OBS reqs to "partial (manual-verify)" even though every one was VERIFIED in the phase VERIFICATION tables. A metadata gap created audit noise, not a delivery gap. +- **VALIDATION.md status fields lagged.** Phases 25/26/27 still carry `status: planned` / `wave_0_complete:false` despite GREEN suites; 28/29 needed an explicit post-exec flip. The nyquist flag was true, but the doc-status fields drifted from reality. +- **16 live-UAT items can't run in CI.** SSE/EventSource behavior and visual rendering need a real browser/live socket; correct to defer, but the true acceptance bar (live dashboard) is unexercised by automated tests. + +### Patterns Established +- **Spike-first for the highest-risk infra phase**, with the concurrency invariant (single producer, no cross-thread queue access) designed before implementation. +- **Audit → insert cleanup phase → re-audit** as a closing loop: convert non-blocking audit warnings into a scoped decimal phase rather than shipping them as raw debt. +- **Static-template TestClient assertions** as the RED scaffold for no-Node frontend phases (assert on the served HTML/JS), keeping frontend behavior test-guarded without a browser. + +### Key Lessons +1. **Give inserted decimal phases a top-level ROADMAP checkbox at insertion time** (`- [ ] **Phase N.M: ...**`), so `roadmap_complete` tracks them and a future autonomous run does not re-execute finished work. +2. **Put `requirements:` in every plan's SUMMARY frontmatter.** The milestone audit cross-references it as one of three sources; omitting it manufactures "partial" statuses for fully-delivered requirements. +3. **Flip VALIDATION.md status post-execution, not just the nyquist flag** — stale `status: planned` on a GREEN phase reads as incomplete to anyone (or any tool) scanning frontmatter. + +### Cost Observations +- Model mix: not tracked this milestone. +- Notable: the close ran the audit refresh with two parallel subagents (integration checker + verification aggregator), then synthesized; faster than serial reading and kept the main context lean. + +--- + +## Milestone: v4.2 — Release Readiness + +**Shipped:** 2026-07-03 +**Phases:** 6 (30-35) | **Plans:** 20 | **Tasks:** 45 | **Suite:** 940 passed, 2 skipped + +### What Was Built +- Breakfix hardening: place-order-timeout double-buy latch (`_PossiblyPlaced` marker + guard, HIGH), Amazon WAF auto-solve via the existing 2captcha path with manual-pause fallback preserved, real post-login DOM/URL verification (`_verify_login_generic`) rolled out to all 7 plugins (P30). +- CI & security infrastructure: gitleaks secret-scan CI job + local guard test, repaired CodeQL workflow (retired Node16 actions bumped to `checkout@v6`/`codeql-action@v4`), `.github/dependabot.yml` + all 7 open vulnerability alerts remediated (P31). +- Release automation & community readiness: `pyproject.toml` reconciled to 2.0.0, release-please seeded (manifest-mode, python release-type), README rewritten, SECURITY.md/CODE_OF_CONDUCT.md routed through GitHub Private Vulnerability Reporting (P32). +- Config refactor: canonical `delay_seconds`/`delay_jitter` fields with a legacy back-compat shim; generic per-platform config extension point (`PlatformsConfig(extra="allow")` + `RetailerPlugin.get_platform_config()`) (P33). +- Feature completion: `[plugin]` log tag on every log line + `/api/logs` plugin filter (completes OBS-08); outcome analytics (success-rate, time-to-checkout) over confirmed-order records (P34). +- Audit-fixes & doc-hygiene: SSR remove-button graceful degradation, `last_heartbeat` leak scrubbed from `get_status()`/SSE, dead `escHtml()` removed, v4.0/v4.1 planning-artifact frontmatter reconciled (P35). + +### What Worked +- **Deep code review after verification caught real bugs twice.** A dedicated REVIEW.md pass (distinct from VERIFICATION.md) found and the executor fixed a genuine CRITICAL regression in both P30 (BF-02's marker was originally written before the `test_mode`/monitor_only suppression check — would have permanently latched any item reached under the documented `test_mode:true` default) and P35 (`POST /items/remove` crashed with an unhandled 500 on a malformed multipart field). Both were confirmed fixed by the milestone audit, both covered by dedicated regression tests. +- **Single shared mechanisms held the line again.** `_verify_login_generic` (one login-verification path for all 7 plugins instead of 7 bespoke checks), the single `get_snapshot()` shaping boundary that fixed `last_heartbeat` for both `get_status()` and SSE atomically, and `get_platform_config()` as the one sanctioned per-plugin config-extension mechanism — DRY held under a milestone that touched every plugin file. +- **Debt-inventory-driven milestone scoping.** An exhaustive automated sweep (64 raw findings → 20 code-actionable requirements) converted a loose backlog of seeds/breakfixes/audit-warnings/deferred-subfeatures into a boundaried, closeable milestone with a clean "code-complete + CI-green, live-UAT stays operator debt" definition of done. +- **Sequencing by risk and dependency paid off.** BF-02 (the one HIGH item) shipped first in Phase 30 to close the longest-standing exposure earliest; Phase 31 (CI security) landed before Phase 32 (release-please) so release automation tagged against a working CI signal. + +### What Was Inefficient +- **The milestone's own Nyquist frontmatter lag was not self-applied.** Phase 35 (DH-01/DH-03) reconciled stale `VALIDATION.md` flags for OLDER phases (v4.0 18-24, v4.1 25-27) — but the audit found all 6 of v4.2's own phase `VALIDATION.md` files still read `status:draft` / `nyquist_compliant:false` at audit time, the exact same staleness pattern the milestone itself was closing for others. Doc-hygiene work should include a final pass over the current milestone's own phases, not just older ones. +- **An autonomous policy decision (RH-07) needed explicit operator sign-off.** Resolving the security-contact requirement to "GitHub PVR only, no email" is a reasonable default, but it is a scope/policy call made in the operator's absence — surfaced correctly as tech debt by the audit, but a reminder that CLAUDE.md's approval-gate rule applies to policy substitutions, not just code changes. +- **Real residual risk was accepted, not just deferred, in one place.** BF-02's marker only covers Amazon + BestBuy this milestone; the 5 community plugins remain exposed to the identical double-buy mechanism the HIGH item was created to close. The scope cut was deliberate and pre-declared (all 5 plugins are independently EXPERIMENTAL/selector-unverified), but it is the one item in this milestone that reads as "should not stay open indefinitely" rather than ordinary live-UAT debt. + +### Patterns Established +- **REVIEW.md as a mandatory post-verification step on safety/security-touching phases** — a second, adversarial read of the diff after VERIFICATION.md passes, specifically hunting for ordering bugs (guard-after-effect) and malformed-input crashes that fixture-driven tests don't naturally cover. +- **Inventory-sweep milestone scoping** for debt-closure milestones: enumerate everything outstanding across seeds/todos/audit-warnings/deferred-subfeatures first, then filter to what's CI-verifiable, before writing requirements. +- **Self-referential doc-hygiene check**: when a phase's job is to reconcile stale frontmatter, explicitly check whether the CURRENT milestone's own phases need the same fix, not only prior milestones'. + +### Key Lessons +1. **Run a REVIEW.md-style deep code review after VERIFICATION.md on any phase touching a safety-critical guard or a new unauthenticated input surface** — it found 2 CRITICAL bugs this milestone that automated/fixture-driven verification missed. +2. **When closing a milestone whose own phases include doc-hygiene reconciliation, apply the same reconciliation to the current milestone's phases before calling it done** — otherwise the very next audit finds the milestone did not eat its own dog food. +3. **Flag autonomous-mode policy substitutions (not just code deviations) for explicit operator sign-off** — a working substitute (PVR instead of email) still represents a decision the operator did not make themselves. + +### Cost Observations +- Model mix: not tracked this milestone. +- Notable: milestone scoped via an exhaustive automated inventory sweep (64 raw → 20 code-actionable) rather than fresh feature ideation — an efficient way to convert accumulated cross-milestone debt into a single boundaried, closeable unit of work. + +--- + ## Cross-Milestone Trends ### Cumulative Quality @@ -53,8 +133,12 @@ | v2.0 | 341 passed | Modular core + CredentialStore (no plaintext on disk) | | v3.0 | 548 passed, 2 skipped | Anti-detection + ecosystem + price monitoring | | v4.0 | 755 passed, 2 skipped | Verified checkout + always-on reliability | +| v4.1 | 807 passed, 2 skipped | Dashboard redesign + live SSE observability (zero-Node) | +| v4.2 | 940 passed, 2 skipped | Debt-closure + release-hardening (double-buy guard, CI security, release-please, config harmonization, log/analytics, audit-fixes) | ### Top Lessons (Verified Across Milestones) 1. Prune archived-milestone detail from the live ROADMAP at close — keeps `roadmap.analyze` accurate and ROADMAP constant-size. -2. Single enforcement points (one ABC gate, one RetryPolicy) backed by CI guards beat per-site patches for safety-critical invariants. +2. Single enforcement points (one ABC gate, one RetryPolicy, one shared verification/shaping helper) backed by CI guards beat per-site patches for safety-critical invariants. +3. Keep planning-doc metadata in sync with reality at close: inserted decimal phases need a top-level ROADMAP checkbox, plan SUMMARYs need a `requirements:` field, and VALIDATION status must flip post-exec — including for the CURRENT milestone's own phases, not only older ones — or stale frontmatter misleads both humans and `roadmap.analyze`/audit tooling. +4. A post-verification deep code review (REVIEW.md) is worth running on any phase touching a safety-critical guard or new unauthenticated input surface — it has caught CRITICAL bugs (v4.2 P30, P35) that fixture-driven verification missed. diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 027ead1..f5c32e9 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -12,6 +12,9 @@ - ✅ **v2.0 Modular Core + Cross-Platform UX** — Phases 7-11 (shipped 2026-06-06) - ✅ **v3.0 Resilience + Ecosystem** — Phases 12-17 (shipped 2026-06-10) - ✅ **v4.0 Win-the-Drop (Acquisition Core + Reliability)** — Phases 18-24 (shipped 2026-06-25) +- ✅ **v4.1 Dashboard & Observability** — Phases 25-29.1 (shipped 2026-06-30) +- ✅ **v4.2 Release Readiness** — Phases 30-35 (shipped 2026-07-03) +- 🔄 **v5.0 Real Release & Plugin Ecosystem** — Phases 36-50 (in progress, started 2026-08-02) --- @@ -76,6 +79,305 @@ Audit: `.planning/milestones/v4.0-MILESTONE-AUDIT.md` (status: tech_debt — pre
+
+✅ v4.1 Dashboard & Observability (Phases 25-29.1) — SHIPPED 2026-06-30 + +- [x] Phase 25: Design System (3/3 plans) — 2026-06-25 +- [x] Phase 26: Read-Only API Endpoints (3/3 plans) — 2026-06-27 +- [x] Phase 27: SSE Infrastructure (3/3 plans) — 2026-06-27 +- [x] Phase 28: Frontend Observability Surfaces (4/4 plans) — 2026-06-27 +- [x] Phase 29: SSE Client Wiring (3/3 plans) — 2026-06-28 +- [x] Phase 29.1: v4.1 Tech-Debt Cleanup — SSE stall fallback + uPlot load order + log-dup guard (4/4 plans, INSERTED) — 2026-06-30 + +Full phase detail archived at `.planning/milestones/v4.1-ROADMAP.md`. +Audit: `.planning/milestones/v4.1-MILESTONE-AUDIT.md` (status: tech_debt — 2 low-sev warnings + 16 pre-accepted live-UAT items, no blockers). + +
+ +
+✅ v4.2 Release Readiness (Phases 30-35) — SHIPPED 2026-07-03 + +- [x] Phase 30: Breakfix Hardening (6/6 plans) — 2026-07-02 +- [x] Phase 31: CI & Security Infrastructure (3/3 plans) — 2026-07-02 +- [x] Phase 32: Release Automation & Community Readiness (3/3 plans) — 2026-07-02 +- [x] Phase 33: Config Refactor (2/2 plans) — 2026-07-02 +- [x] Phase 34: Feature Completion (3/3 plans) — 2026-07-03 +- [x] Phase 35: Audit-Fixes & Doc-Hygiene Cleanup (3/3 plans) — 2026-07-03 + +Full phase detail archived at `.planning/milestones/v4.2-ROADMAP.md`. +Audit: `.planning/milestones/v4.2-MILESTONE-AUDIT.md` (status: tech_debt — 20/20 requirements satisfied, no blockers; BF-02 5-plugin propagation gap + operator-gated GitHub Settings items tracked as debt). + +
+ +### 🔄 v5.0 Real Release & Plugin Ecosystem (Phases 36-50) — IN PROGRESS + +- [ ] **Phase 36: Mainline Reconciliation** — `master` becomes the real ShopPyBot and its suite runs in CI for the first time (MAIN-01..07) +- [ ] **Phase 37: Distributable Artifact** — the built wheel actually runs, so publishing one is worth doing (PKG-01..06) +- [ ] **Phase 38: Scanning to Zero** — every scanner reports zero real findings and the checks that produce them are required (SCAN-01..11) +- [ ] **Phase 39: Quality Floor** — lint, format, typecheck, and coverage enforced in CI before the milestone's new code lands (QUAL-01..09) +- [ ] **Phase 40: Public-Repo Readiness** — LICENSE, current README, honest sample config, CODEOWNERS, drift corrected (PUB-01..09) +- [ ] **Phase 41: Live Defect Closure** — control commands report the truth, unattended alerts arrive, dashboard data is current (FIX-01..11) +- [ ] **Phase 42: Plugin Registry Hardening** — a malformed plugin cannot take down start, the dashboard, or `plugins list` (EXT-01, EXT-02) +- [ ] **Phase 43: Plugin Roots, Precedence & API Version Gate** — a user-writable second root, bundled-wins collisions, version enforcement (EXT-03..05) +- [ ] **Phase 44: Provenance, Load-Boundary Integrity & Run Lock** — every managed plugin says where it came from and drifted files do not execute (EXT-06..08) +- [ ] **Phase 45: Community Plugin Parity + Pre-Transfer Arming Gate** — 5 community plugins reach the Amazon/BestBuy safety floor; the shared checkout gate is built once (PAR-01..06) +- [ ] **Phase 46: Trust Tiers & Capability Reduction** — third-party plugins ship disarmed for checkout and scoped for credentials (EXT-09, EXT-10) +- [ ] **Phase 47: Fetch, Pre-Flight, Install & Consent** — `plugins install` pinned by commit SHA, refused statically, gated by typed consent (EXT-11..13) +- [ ] **Phase 48: Plugin Lifecycle — update, remove, verify, outdated** — mutations re-consent, removal prints a rotation checklist (EXT-14) +- [ ] **Phase 49: Trust Documentation, Registry & Vocabulary Guard** — nothing shipped calls third-party plugins safe (EXT-15..17) +- [ ] **Phase 50: Ops Hardening & UAT Consolidation** — bounded price history, one honest operator verification checklist (OPS-01..02, UAT-01..06) + +--- + +## Phase Details (v5.0) + +### Phase 36: Mainline Reconciliation + +**Goal**: `master` is the real ShopPyBot — every v4.1 and v4.2 artifact is on the default branch, and the suite those milestones claimed runs green in CI for the first time. +**Depends on**: Nothing (first phase of the milestone; every other phase is undurable until this lands) +**Requirements**: MAIN-01, MAIN-02, MAIN-03, MAIN-04, MAIN-05, MAIN-06, MAIN-07 +**Success Criteria** (what must be TRUE): + + 1. A CI run on `master` schedules jobs and reports the full v4.1+v4.2 suite green on both `ubuntu-latest` and `windows-latest` — where today 81 consecutive runs have scheduled zero jobs and produced no logs. + 2. `gitleaks.yml`, `release-please.yml`, the dashboard, and `httpx==0.28.1` are all present on `master` at HEAD. + 3. `gh pr list` shows #8, #11, #12 and #15 through #20 all resolved: #11, #12 and the Dependabot set merged in an order that did not require re-resolving the `ci.yml` collision between #19 and #20; #8 closed unmerged with a recorded reason. + 4. Each of the 4 local commits absent from PR #11's head carries a recorded include-or-exclude decision, and `git log master` matches that decision rather than reflecting a conflict-resolution side effect. + +**Plans**: 5 plans (strictly sequential, waves 1 through 5; `strict: true` branch protection makes parallel merges impossible) +Plans: + +- [x] 36-01-PLAN.md: rollback tag `pre-v5-mainline`, merge PR #12, close PR #8 with a recorded reason (MAIN-05, MAIN-06) +- [x] 36-02-PLAN.md: absorb PR #11 head divergence without force-push, record MAIN-03 per SHA, union-resolve the two conflicts, full local suite gate (MAIN-02, MAIN-03) +- [ ] 36-03-PLAN.md: plain-push, merge PR #11 as a merge commit, verify the union pins, workflow artifacts, SHA ancestry and a real CI run on master (MAIN-01..04) +- [ ] 36-04-PLAN.md: resolve the pip Dependabot set #16, #17, #15 with a direction check before each merge (MAIN-07) +- [ ] 36-05-PLAN.md: serialize the ci.yml set #18, #19, #20, then the phase gate and Phase 38 handoff (MAIN-01, MAIN-07) + +**Planning corrections** (verified live 2026-08-02, these supersede the success criteria above where they conflict): + + - Criterion 1's "81 consecutive zero-job runs" premise is STALE. Master's CI is already green today on `e98ec83` and `4123059`, 755 passed / 2 skipped on both runners. MAIN-01's real remaining question is whether the LARGER merged suite (roughly 939 tests) stays green under a ci.yml already proven correct on the smaller input. + - Criterion 4's "4 local commits" count is STALE. It was 8 at discuss time, 9 at research time, and 11 at planning time. The plans re-derive the list fresh at execution time and never hardcode a count. + - `master` ALREADY has branch protection (`CodeQL`, `test (windows-latest)`, `test (ubuntu-latest)`, `strict: true`, `enforce_admins: false`), which CONTEXT.md did not anticipate. The plans work within it and change nothing; Phase 38 still owns protection changes. + +**Risk**: Highest-risk phase in the milestone. Small in requirement count, large in blast radius — a 263-commit merge whose CI has never executed, so the first green run is also the first signal that the merge is correct. Plan for verification room, not just merge mechanics. + +### Phase 37: Distributable Artifact + +**Goal**: An installed wheel is a working ShopPyBot, so release-please publishing one becomes worth doing. +**Depends on**: Phase 36 (the wheel CI job needs a `ci.yml` that compiles) +**Requirements**: PKG-01, PKG-02, PKG-03, PKG-04, PKG-05, PKG-06 +**Success Criteria** (what must be TRUE): + + 1. A clean-environment install of the built wheel starts `shoppybot web` without a `StaticFiles` `RuntimeError`, and a named sound file resolves from the installed package. + 2. Installing the wheel into an empty virtualenv, with no `requirements.txt` available, produces no `ModuleNotFoundError` for `websockets`, `starlette`, `httpx`, or `requests` on any exercised path. + 3. A CI job performs criteria 1 and 2 on every push and fails the build if either regresses. + 4. `bundled_plugins_dir()` called from an installed wheel returns a directory containing the 7 bundled plugins, and that result is recorded as the input Phase 43 (EXT-03) depends on. + 5. `requirements.txt` pins neither `selenium` nor `webdriver-manager`, and no Dependabot alert references either. + +**Plans**: TBD +**Note**: PKG-06 is a hard gate on Phase 43. If the bundled root does not survive a wheel install, the `importlib.resources` fix belongs here, not to workstream H. + +### Phase 38: Scanning to Zero + +**Goal**: Every security scanner attached to the repo reports zero outstanding real findings, and the checks that produce them are required to merge. +**Depends on**: Phase 36 (several of these target workflows that do not exist on `master` until PR #11 lands) +**Requirements**: SCAN-01, SCAN-02, SCAN-03, SCAN-04, SCAN-05, SCAN-06, SCAN-07, SCAN-08, SCAN-09, SCAN-10, SCAN-11 +**Success Criteria** (what must be TRUE): + + 1. The Dependabot, CodeQL, and secret-scanning queues each show zero open real findings: the 7 dependency alerts closed, the two `logger.py` clear-text alerts and the 3 production url-sanitization alerts resolved, the 19 test-file url-sanitization alerts showing as deliberately dismissed with a recorded reason, and secret scanning running with non-provider patterns and validity checks enabled. + 2. A pull request against `master` cannot merge without gitleaks, CodeQL, and the test matrix passing, and the rule applies to administrators. + 3. `.github/workflows/` on `master` contains no `disabled_manually` CodeQL workflow, and `origin/dev` with its 4 rotted workflows has a recorded disposition. + 4. Every third-party action in every workflow is referenced by a 40-character commit SHA, and `ci.yml` declares an explicit `permissions` block. + +**Plans**: TBD + +### Phase 39: Quality Floor + +**Goal**: The code standards CLAUDE.md already states are mechanically enforced on every push, before this milestone's remaining eleven phases write new code. +**Depends on**: Phase 36 (CI must compile), Phase 38 (new CI jobs follow the SHA-pinning and permissions conventions established there) +**Requirements**: QUAL-01, QUAL-02, QUAL-03, QUAL-04, QUAL-05, QUAL-06, QUAL-07, QUAL-08, QUAL-09 +**Success Criteria** (what must be TRUE): + + 1. A push introducing a lint, format, or type error fails CI; pre-existing findings are either fixed or recorded in a checked-in baseline that the build reads. + 2. A CI run reports a coverage number and the starting baseline is recorded in-repo. + 3. No `except Exception: pass` site remains under the production tree — each of the 11 either handles the error or lets it propagate. + 4. The two tests that skip when `config.yml` is absent fail loudly instead, including the plaintext-secrets security check, and a full suite run emits no never-awaited-coroutine warning. + 5. `amazon_bot.py` and `bestbuy_bot.py` no longer exist, and every file over 300 lines and function over 30 lines is either under the limit or listed in a recorded exceptions file with rationale. + +**Plans**: TBD + +### Phase 40: Public-Repo Readiness + +**Goal**: A stranger landing on the repository gets an accurate, licensed, current picture of what ShopPyBot is and how to run it. +**Depends on**: Phase 36 (docs are written against what is on `master`) +**Requirements**: PUB-01, PUB-02, PUB-03, PUB-04, PUB-05, PUB-06, PUB-07, PUB-08, PUB-09 +**Success Criteria** (what must be TRUE): + + 1. The repository root carries a LICENSE file and GitHub displays the license on the repo page. + 2. Following README's Prerequisites and Setup on a clean machine gets the bot running via nodriver, with no step mentioning `chromedriver` or `selenium.driver_path`. + 3. `sample.config.yml` round-trips through `AppConfig` with every declared section present including `debug.monitor_only`, and advertises no key that nothing reads. + 4. `_deprecated/` is absent from `master`, CODEOWNERS is present so code-owner review can be required, and CLAUDE.md and SECURITY.md describe the current architecture and the already-enabled PVR state. + 5. Every drift item from the 2026-08-01 audit reads correctly (dashboard section count, the web launcher command, banner colour values, cold-load theme default, MC-3's polling description, AmazonPlugin's registry difficulty), and SEED-001's destructive history rewrite is retired as a recorded decision citing the one test-fixture false positive found across 964 commits. + +**Plans**: TBD +**Note**: LICENSE (PUB-01) is where Phase 49's formal liability language lands, so this phase precedes it. + +### Phase 41: Live Defect Closure + +**Goal**: The defects a real run actually hits are gone — control commands report the truth, the alerts that exist only for unattended operation arrive, and the dashboard shows current data. +**Depends on**: Phase 36, Phase 39 (new code lands under the enforced standard) +**Requirements**: FIX-01, FIX-02, FIX-03, FIX-04, FIX-05, FIX-06, FIX-07, FIX-08, FIX-09, FIX-10, FIX-11 +**Success Criteria** (what must be TRUE): + + 1. Pressing Start on the dashboard when the bot loop cannot survive startup surfaces a real failure in the UI, instead of a 200 over a dead loop. + 2. A headless run with no TTY, and a run where `sys.stdin` is None, neither busy-spins nor force-fires manual-intervention gates; a listener failure is reported rather than dying silently; and shutdown completes in seconds rather than stalling `asyncio.run()` teardown for up to 300. + 3. Saving `test_mode` from the dashboard either changes the running process's behaviour or reports that it did not, and the test covering that write patches the real module path and asserts the write it claims to test. + 4. A `plugin_parked` event and a `health_degraded` event each produce a delivered Discord, email, and SMS message, visually distinguishable from a successful purchase, with all six `action` values documented and the plugin name escaped and length-capped on every surface. + 5. Starting on a host without Chrome prints an actionable diagnosis instead of every plugin failing silently, and the Confirmed Orders view updates without a page reload. + +**Plans**: TBD +**UI hint**: yes +**Note**: FIX-08 (escape and cap the plugin name) must land before Phase 47 makes that value attacker-controlled. + +### Phase 42: Plugin Registry Hardening + +**Goal**: A malformed or hostile plugin cannot take down bot start, the dashboard render, `plugins list`, or `run_plugin`, and the template contributors copy stops teaching the checkout bypass. +**Depends on**: Phase 39 (H1 in `research/SUMMARY.md`; the only H phase independently valuable if the rest of H is cut) +**Requirements**: EXT-01, EXT-02 +**Success Criteria** (what must be TRUE): + + 1. A plugin that omits `domain_patterns`, or raises inside `__init__`, is skipped with one discovery-time warning while every other plugin still loads — `shoppybot run` starts, the dashboard renders, and `plugins list` prints its table. + 2. `/api/analytics` and `run_plugin` survive that same malformed plugin without the supervisor treating it as a crash and entering a backoff-restart loop. + 3. Two consecutive `plugins list` runs on the same machine discover plugins in the same order. + 4. `plugins/example_plugin.py` places its order through `place_order_guarded()`, so a copy of the canonical template honours `test_mode` and writes the BF-02 place-order marker. + +**Plans**: TBD +**Note**: Routing determinism (`route()` picking the same plugin across runs) belongs to PAR-06 in Phase 45; this phase covers discovery-order stability only. + +### Phase 43: Plugin Roots, Precedence & API Version Gate + +**Goal**: Plugins live in two clearly separated roots, bundled code always wins a collision, and a plugin that does not match the supported API version is refused rather than loaded. +**Depends on**: Phase 42 (multi-root discovery is where malformed third-party plugins first arrive), Phase 37 (PKG-06 answers whether the bundled root survives a wheel install) +**Requirements**: EXT-03, EXT-04, EXT-05 +**Success Criteria** (what must be TRUE): + + 1. A `.py` file placed in the user plugin directory under `data_dir()` is discovered and loaded, and `SHOPBOT_DATA_DIR` relocates that directory in tests. + 2. A user-root file named after a bundled plugin, or declaring a domain a bundled plugin claims, is refused at load with a message naming the conflict, and the bundled plugin still loads. + 3. `plugins list` and the dashboard show, for every plugin, whether it is bundled or third-party. + 4. A plugin declaring an API version below the supported minimum is refused; one declaring an older-but-supported version loads with a warning. + 5. No production code computes a plugin path outside `core/paths.py`. + +**Plans**: TBD +**UI hint**: yes +**Research flag**: Blocked on PKG-06's factual answer. If `bundled_plugins_dir()` does not resolve from an installed wheel, the `importlib.resources` fix is Phase 37 work, not this phase's. + +### Phase 44: Provenance, Load-Boundary Integrity & Run Lock + +**Goal**: Every plugin in the user root can say where it came from, a file that changed since it was recorded does not execute, and lifecycle commands can tell the truth about whether a bot is running. +**Depends on**: Phase 43 (needs the writable root and the origin field to join against) +**Requirements**: EXT-06, EXT-07, EXT-08 +**Success Criteria** (what must be TRUE): + + 1. An `installed.json` beside the user-root plugins records, per plugin, the source and URL, the numeric owner ID, the resolved 40-character commit SHA, an independently computed content SHA-256, the declared API version, and the consent that authorised it — and survives an interrupted write without corrupting. + 2. Changing one byte of a managed plugin file causes it to be refused on the next registry construction, including an ordinary dashboard page render rather than only a bot start, and `plugins list` reports it as `modified`. + 3. An unmanifested file dropped into the user root is refused and logged; the same file placed in the bundled root still loads, so drop-in development is unchanged. + 4. `shoppybot status` answers whether a bot is running in another process, and that answer degrades honestly (advisory staleness) when the holding process was killed. + +**Plans**: TBD +**Note**: The manifest informs discovery; it does not authorise. No shipped wording may describe the load check as an authorization boundary — whoever can drop a file into that directory can write the manifest beside it. + +### Phase 45: Community Plugin Parity + Pre-Transfer Arming Gate + +**Goal**: The 5 community plugins reach the same safety floor as Amazon and BestBuy, and the single per-plugin checkout gate that Phase 46 extends to third-party code exists and is proven on plugins the maintainer owns. +**Depends on**: Phase 44 (PAR-06's bundled-before-third-party ordering needs the origin field from Phase 43) +**Requirements**: PAR-01, PAR-02, PAR-03, PAR-04, PAR-05, PAR-06 +**Success Criteria** (what must be TRUE): + + 1. A place-order-stage timeout on Walmart, Target, GameStop, NewEgg, or SquareEnix latches the item and refuses the retry, exactly as Amazon and BestBuy already do — closing the residual double-buy exposure carried out of v4.2. + 2. A hung selector in a community plugin aborts that step within its own budget instead of blocking the plugin loop, and the resulting error log names the checkout stage it failed in. + 3. With `monitor_only` set, no community plugin reaches its checkout entry point, enforced at the one pre-transfer gate that Phase 46 extends — not at a second gate that can diverge from it. + 4. Confirmation detection reads the correct tab for every community plugin. + 5. With two plugins whose `domain_patterns` overlap, `route()` returns the same plugin on every run on the same machine. + +**Plans**: TBD +**Review**: REVIEW.md deep-review pass required (RETROSPECTIVE.md lesson 4 — this phase touches a safety-critical guard). The review must state, for every shipped control, one sentence naming what it does not stop. + +### Phase 46: Trust Tiers & Capability Reduction + +**Goal**: A third-party plugin cannot spend money or reach a credential it was not scoped to, unless the operator has explicitly armed it. +**Depends on**: Phase 45 (shared arming mechanism), Phase 44 (the manifest's trust field) +**Requirements**: EXT-09, EXT-10 +**Success Criteria** (what must be TRUE): + + 1. A third-party plugin is refused at the pre-transfer gate even with the global `monitor_only` off, until an explicit typed opt-in arms it; the armed state is recorded in the manifest and survives a restart. + 2. A bundled plugin's behaviour is unchanged by the third-party disarm default. + 3. A plugin requesting credentials receives only its own platform's keys, and a test proves a third-party plugin never receives the CVV. + 4. Every user-facing string describing the arming gate carries the sentence naming what it does not stop — a purchase performed by the plugin inside `check_availability`, which runs before the gate. + +**Plans**: TBD +**Note**: This phase must ship before Phase 47 so no released state exists in which a freshly installed stranger's plugin can buy by default. + +### Phase 47: Fetch, Pre-Flight, Install & Consent + +**Goal**: `plugins install` can bring a stranger's plugin onto the machine — pinned, statically screened, and consented to — without ever importing the candidate to decide whether to install it. +**Depends on**: Phase 46 (the disarm default must already be true), Phase 44 (the manifest to write into and the run lock to warn from) +**Requirements**: EXT-11, EXT-12, EXT-13 +**Success Criteria** (what must be TRUE): + + 1. `plugins install /[@ref]` resolves the ref to a 40-character commit SHA, fetches at that SHA, and records the SHA plus an independently computed content hash; a branch or tag name is never what gets stored. + 2. A candidate that fails to parse, declares no `RetailerPlugin` subclass, imports outside the allowlist, or contains an invisible codepoint, bidi override, variation selector, or Private Use Area character outside strings and comments is refused before any file is written — and nothing about the candidate was imported to reach that verdict. + 3. The install prompt shows `github.com//` and the numeric owner ID in a handful of concrete lines and requires the plugin name typed back; no bare `--yes` exists, and non-interactive install requires `--expect-sha256`. + 4. A successful install prints that a restart is required, and the newly installed plugin is disarmed for checkout. + 5. A rate-limited or unreachable GitHub produces an explicit failure, never a silent fallback to a cache or an alternate source. + +**Plans**: TBD +**Review**: REVIEW.md deep-review pass required (RETROSPECTIVE.md lesson 4 — this phase opens the milestone's new unauthenticated input surface). Two non-negotiable criteria: one sentence per shipped control naming what it does not stop, and confirmation that no shipped artifact describes third-party plugins as sandboxed, isolated, curated, verified, or safe. +**Research flag**: The consent prompt copy is an acceptance criterion, not an implementation detail. Re-read the consent-fatigue evidence in `research/SUMMARY.md` before writing it. + +### Phase 48: Plugin Lifecycle — update, remove, verify, outdated + +**Goal**: An installed plugin can be changed or taken off the machine with the same honesty the install had, and removal tells the operator what it did not undo. +**Depends on**: Phase 47 (update is install-with-a-prior-record and cannot exist before install) +**Requirements**: EXT-14 +**Success Criteria** (what must be TRUE): + + 1. `plugins update ` re-prompts for consent on any SHA or content change, shows what changed, and treats an owner-ID change or a capability escalation as a new install rather than an update. + 2. `plugins outdated` reports and can never install, and no scheduler, timer, or startup hook can reach the update path. + 3. `plugins verify` re-checks every managed file against its recorded hash and names each mismatch. + 4. `plugins remove ` refuses by default while a run lock is held and refuses outright while a place-order marker is set, and on success prints a credential and session rotation checklist whose referenced commands all exist. + +**Plans**: TBD +**Note**: Covered by Phase 47's REVIEW.md scope — the same input surface. Re-run both REVIEW criteria against `update` and `remove` before closing this phase. + +### Phase 49: Trust Documentation, Registry & Vocabulary Guard + +**Goal**: Nothing ShopPyBot ships describes third-party plugins as safe, and the disclaimer a user needs is at the point where the decision is made. +**Depends on**: Phase 48, Phase 40 (the LICENSE the liability language lands beside) +**Requirements**: EXT-15, EXT-16, EXT-17 +**Success Criteria** (what must be TRUE): + + 1. The third-party disclaimer appears at the point of decision — in the install prompt and beside every third-party row in `plugins list` — and in SECURITY.md, README, and PLUGIN_DEV.md, stating plainly that third-party installs receive no review of any kind and that no revocation mechanism exists. + 2. SECURITY.md states that its per-platform risk assessment applies to merged plugins only. + 3. A CI job fails the build when any shipped artifact — code, docstring, variable name, doc, CLI help, release note, or registry row — calls third-party plugins sandboxed, isolated, curated, verified, or safe. + 4. An in-repo machine-readable registry is the source of truth, CI generates the JSON index and the markdown table from it, and the hand-maintained wiki page is retired rather than left to rot (closes REG-01, outstanding across two milestones). + +**Plans**: TBD +**Milestone gate**: EXT-15 gates milestone completion. The disclaimer is the precondition that makes the plugin manager defensible; the milestone does not close without it. + +### Phase 50: Ops Hardening & UAT Consolidation + +**Goal**: The operational data path stays fast and bounded, and the outstanding-verification picture is one honest operator checklist instead of 88 scattered archive files. +**Depends on**: Phase 36 (UAT-03 re-runs the items PR #11 and #12 unblock; UAT-06 corrects a claim only Phase 36 can make true), Phase 49 +**Requirements**: OPS-01, OPS-02, UAT-01, UAT-02, UAT-03, UAT-04, UAT-05, UAT-06 +**Success Criteria** (what must be TRUE): + + 1. A per-item price-history chart loads without a full scan of `price_history`, and the table stops growing without bound under a stated, applied retention policy — against a table carrying weeks of the roughly 15k-rows-per-day a 5-item watch produces. + 2. 29-HV-5 and 29.1-UAT-2 each carry a recipe a person can actually follow to reproduce the condition they test. + 3. The 8 UAT items that PR #11 and PR #12 unblock carry recorded verdicts. + 4. One operator checklist file lists every remaining live-environment item with a stated acceptance bar, and no verdict lives only inside an archived phase artifact. + 5. The `audit-uat` archive blindspot is documented in-repo, and the "CI-green" claim in the v2.0 through v4.2 archives is corrected. + +**Plans**: TBD + --- ## Progress @@ -86,9 +388,55 @@ Audit: `.planning/milestones/v4.0-MILESTONE-AUDIT.md` (status: tech_debt — pre | v2.0 Modular Core + Cross-Platform UX | 7-11 | 20/20 | ✅ Shipped | 2026-06-06 | | v3.0 Resilience + Ecosystem | 12-17 | 21/21 | ✅ Shipped | 2026-06-10 | | v4.0 Win-the-Drop | 18-24 | 29/29 | ✅ Shipped | 2026-06-25 | +| v4.1 Dashboard & Observability | 25-29.1 | 20/20 | ✅ Shipped | 2026-06-30 | +| v4.2 Release Readiness | 30-35 | 20/20 | ✅ Shipped | 2026-07-03 | +| v5.0 Real Release & Plugin Ecosystem | 36-50 | 0/TBD | 🔄 In progress | — | + +All requirements satisfied across v1 (44) + v2.0 (22) + v3.0 (18) + v4.0 (17) + v4.1 (16) + v4.2 (20). Per-milestone requirement detail in `.planning/milestones/v*-REQUIREMENTS.md`. v5.0 carries 84 requirements across 15 phases (36-50); per-requirement mapping in `.planning/REQUIREMENTS.md` → Traceability. + +### v5.0 phase-status detail + +| Phase | Plans Complete | Status | Completed | +|-------|----------------|--------|-----------| +| 36. Mainline Reconciliation | 2/5 | In Progress| | +| 37. Distributable Artifact | 0/TBD | Not started | - | +| 38. Scanning to Zero | 0/TBD | Not started | - | +| 39. Quality Floor | 0/TBD | Not started | - | +| 40. Public-Repo Readiness | 0/TBD | Not started | - | +| 41. Live Defect Closure | 0/TBD | Not started | - | +| 42. Plugin Registry Hardening | 0/TBD | Not started | - | +| 43. Plugin Roots, Precedence & API Version Gate | 0/TBD | Not started | - | +| 44. Provenance, Load-Boundary Integrity & Run Lock | 0/TBD | Not started | - | +| 45. Community Plugin Parity + Pre-Transfer Arming Gate | 0/TBD | Not started | - | +| 46. Trust Tiers & Capability Reduction | 0/TBD | Not started | - | +| 47. Fetch, Pre-Flight, Install & Consent | 0/TBD | Not started | - | +| 48. Plugin Lifecycle — update, remove, verify, outdated | 0/TBD | Not started | - | +| 49. Trust Documentation, Registry & Vocabulary Guard | 0/TBD | Not started | - | +| 50. Ops Hardening & UAT Consolidation | 0/TBD | Not started | - | + +--- + +## v5.0 Sequencing Invariants + +These are the seven hard constraints from `REQUIREMENTS.md`, mapped onto phase numbers. Reordering phases without re-checking these breaks the milestone. + +| # | Constraint | Satisfied by | +|---|------------|--------------| +| 1 | A before E — SCAN targets workflows not on `master` until PR #11 lands | Phase 36 → Phase 38 | +| 2 | B before release-please is worth running | Phase 37 precedes any release cut | +| 3 | PKG-06 before EXT-03 | Phase 37 → Phase 43 | +| 4 | EXT-01 before every other EXT | Phase 42 is the first EXT phase | +| 5 | EXT-09 before EXT-11 — the disarm default must be true before install ships | Phase 46 → Phase 47 | +| 6 | PAR-03 and EXT-09 are one mechanism | Phase 45 builds the pre-transfer arming gate; adjacent Phase 46 extends it. One gate, not two | +| 7 | EXT-15 gates milestone completion, not phase ordering | Phase 49 is a milestone gate; Phase 50 may still follow it | + +**Additional derived orderings (not hard constraints, but deliberate):** -All requirements satisfied across v1 (44) + v2.0 (22) + v3.0 (18) + v4.0 (17). Per-milestone requirement detail in `.planning/milestones/v*-REQUIREMENTS.md`. +- Phase 39 (Quality Floor) precedes Phases 41-50 so the milestone's new code is written under an enforced standard rather than retrofitted to one. +- Phase 41's FIX-08 (escape and length-cap the plugin name) precedes Phase 47, which is what makes that value attacker-controlled. +- Phase 40's LICENSE precedes Phase 49's liability language, which lands beside it. +- The H workstream's internal order (Phases 42, 43, 44, 46, 47, 48, 49 = H1-H7) is the reconciled order from `research/SUMMARY.md`. It is researched, not re-derived. --- -*Last updated: 2026-06-25 — v4.0 Win-the-Drop shipped; ready for next milestone (`/gsd:new-milestone`).* +*Last updated: 2026-08-02 — v5.0 Real Release & Plugin Ecosystem roadmap created (Phases 36-50, 84 requirements mapped, 100% coverage). Next: `/gsd:plan-phase 36`.* diff --git a/.planning/STATE.md b/.planning/STATE.md index 05bd245..219ca16 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -1,16 +1,16 @@ --- gsd_state_version: 1.0 -milestone: v4.0 -milestone_name: Win-the-Drop -status: Awaiting next milestone -last_updated: "2026-06-25T01:32:12.122Z" -last_activity: 2026-06-25 — Milestone v4.0 completed and archived +milestone: v5.0 +milestone_name: Real Release & Plugin Ecosystem +status: executing +last_updated: "2026-08-02T18:19:06.904Z" +last_activity: 2026-08-02 progress: - total_phases: 7 - completed_phases: 7 - total_plans: 29 - completed_plans: 29 - percent: 100 + total_phases: 15 + completed_phases: 0 + total_plans: 5 + completed_plans: 1 + percent: 0 --- # ShopPyBot — State @@ -20,81 +20,152 @@ progress: **Core Value**: Drop-in plugin framework — community adds retail platform integrations via a single Python file in `plugins/`; no core changes required. **Project**: ShopPyBot -**Milestone**: v4.0 Win-the-Drop (Acquisition Core + Reliability) -**Total Phases**: 7 (Phases 18-24) -**Total Requirements**: 17 +**Milestone**: v5.0 Real Release & Plugin Ecosystem (Phases 36-50) +**Total Phases**: 15 (Phases 36-50) +**Total Requirements**: 84 (MAIN-01..07, PKG-01..06, PUB-01..09, FIX-01..11, SCAN-01..11, QUAL-01..09, PAR-01..06, EXT-01..17, OPS-01..02, UAT-01..06) --- ## Current Position -Phase: Milestone v4.0 complete -Plan: — -Status: Awaiting next milestone -Last activity: 2026-06-25 — Milestone v4.0 completed and archived +Phase: 36 (Mainline Reconciliation) — EXECUTING +Plan: 3 of 5 +Status: Ready to execute 36-03 (push PR #11 head, merge PR #11, verify CI on master) +Last activity: 2026-08-02 -- Phase 36 Plan 02 complete (divergence absorbed by merge, both conflicts union-resolved, merged tree green at 961 passed / 0 failed across 963 collected; nothing pushed) + +**Carry into 36-03:** +- Local `chore/v4.0-milestone-close` tip is unpushed and ready for a **plain, fast-forward-safe push**. `origin/chore/v4.0-milestone-close` at `e2f2695` is a proven ancestor of local HEAD, so no force is needed and none is permitted. +- Conflict-resolution merge commit: `635c1d3be8ba015a9da58b2242e69038a5859016` (parents `7875a01` branch, `36f75c7` master). +- **Use `/mingw64/bin/git`, not bare `git`.** The rtk shell hook was confirmed live to drop merge commits from `git log ` output. Plan 36-03 task 2's per-SHA ancestry proof reads from a range query and is directly exposed. +- `36-COMMIT-DISPOSITION.md` holds the 21-SHA MAIN-03 record and ends with `Post-merge ancestry verification: pending (plan 03 task 2)`. Plan 36-03 task 2 owns flipping that line. +- `required_status_checks.strict: true` on master. If master moves before the PR #11 merge, an `update-branch` is required first. +- Expected non-blocker: the Actions allowlist is not widened for `gitleaks-action` and `release-please-action`, so those two workflows arrive on master with PR #11 and are expected to fail at action-resolution time. Phase 38 scope; record it, do not treat it as a merge defect or a fix-forward trigger. ## Phase Status | Phase | Goal Summary | Status | Reqs | |-------|-------------|--------|------| -| 18 — Safety Gate + Config Foundation | monitor-only mode + place_order_guarded() ABC + CheckoutConfig schema | Complete | BUY-01, BUY-02 | -| 19 — DB Schema + Confirmation Detection | order_id/confirmed_at columns + core/confirmation.py; purchased only on real order | Complete | BUY-03, BUY-04 | -| 20 — Checkout Profile + Form-Fill | 9-key CredentialStore profile; BestBuy + Amazon form-fill; CVV getpass-only | Complete | BUY-07 | -| 21 — Per-Step Timeouts + Unified Retry + Cart-Retry | core/retry.py RetryPolicy; per-step asyncio.timeout; idempotent cart-retry | Complete | BUY-05, BUY-06, REL-08 | -| 22 — Supervisor + Browser Relaunch + Server Safety | per-coroutine supervision; failure budget; full relaunch sequence; DB read isolation; SIGTERM bridge | Complete | REL-01, REL-02, REL-03, REL-05, REL-06, SRV-02 | -| 23 — Encrypted Session Persistence | core/session_store.py Fernet cookies; CDP restore path; replaces Phase 22 stub | Complete | REL-04 | -| 24 — Health Surface + Server Safety | core/health.py HealthRegistry; get_status() expansion; health_degraded alert; pygame headless guard | Complete | REL-07, SRV-01 | +| 36 — Mainline Reconciliation | `master` becomes the real ShopPyBot; the v4.1+v4.2 suite runs in CI for the first time | In Progress (2/5 plans; MAIN-05, MAIN-06 done; MAIN-02, MAIN-03 resolved locally, pending proof on master) | MAIN-01..07 | +| 37 — Distributable Artifact | The built wheel actually runs; truthful dependency declaration; PKG-06 answers EXT-03's blocker | Not started | PKG-01..06 | +| 38 — Scanning to Zero | Dependabot/CodeQL/secret-scanning queues to zero real findings; required checks + branch protection | Not started | SCAN-01..11 | +| 39 — Quality Floor | Lint, format, typecheck, coverage enforced in CI before the milestone's new code lands | Not started | QUAL-01..09 | +| 40 — Public-Repo Readiness | LICENSE, nodriver README, honest sample config, CODEOWNERS, drift corrected, SEED-001 retired | Not started | PUB-01..09 | +| 41 — Live Defect Closure | Control commands report the truth; unattended alerts arrive; dashboard data is current | Not started | FIX-01..11 | +| 42 — Plugin Registry Hardening (H1) | A malformed plugin cannot take down start, the dashboard, `plugins list`, or `run_plugin` | Not started | EXT-01, EXT-02 | +| 43 — Plugin Roots, Precedence & API Version Gate (H2) | User-writable second root, bundled-wins collisions, API version enforced at load | Not started | EXT-03, EXT-04, EXT-05 | +| 44 — Provenance, Load-Boundary Integrity & Run Lock (H3) | Managed plugins carry provenance; drifted files do not execute; lifecycle commands tell the truth | Not started | EXT-06, EXT-07, EXT-08 | +| 45 — Community Plugin Parity + Pre-Transfer Arming Gate (G) | 5 community plugins reach the Amazon/BestBuy safety floor; the shared checkout gate is built once | Not started | PAR-01..06 | +| 46 — Trust Tiers & Capability Reduction (H4) | Third-party plugins disarmed for checkout by default; per-platform credential scoping; never the CVV | Not started | EXT-09, EXT-10 | +| 47 — Fetch, Pre-Flight, Install & Consent (H5) | `plugins install` pinned by commit SHA, statically screened, gated by typed consent | Not started | EXT-11, EXT-12, EXT-13 | +| 48 — Plugin Lifecycle (H6) | `update`/`remove`/`verify`/`outdated`; re-consent on change; removal prints a rotation checklist | Not started | EXT-14 | +| 49 — Trust Documentation, Registry & Vocabulary Guard (H7) | Nothing shipped calls third-party plugins safe; machine-readable registry retires the wiki table | Not started | EXT-15, EXT-16, EXT-17 | +| 50 — Ops Hardening & UAT Consolidation | Bounded, indexed price history; one honest operator verification checklist | Not started | OPS-01..02, UAT-01..06 | --- ## Performance Metrics -**Plans completed**: 29 of 29 -**Requirements completed**: 17 of 17 (BUY-01..07, REL-01..08, SRV-01, SRV-02) -**Phases completed**: 7 of 7 +**Plans completed**: 2 of TBD +**Requirements completed**: 2 of 84 (MAIN-05, MAIN-06; MAIN-02 and MAIN-03 resolved locally, counted once proven on `master` in 36-03) +**Phases completed**: 0 of 15 **Blockers resolved**: 0 +| Phase | Plan | Duration | Tasks | Files | +|-------|------|----------|-------|-------| +| 36 | 01 | 13min | 3 | 2 | +| 36 | 02 | 16min | 3 | 7 | + --- ## Accumulated Context -### Key Decisions Logged +### Sequencing Invariants (v5.0 — carry into planning) + +The seven hard constraints from REQUIREMENTS.md, mapped onto phase numbers. Reordering +phases without re-checking these breaks the milestone. + +- **Phase 36 before Phase 38.** Several SCAN requirements target workflows that do not exist on `master` until PR #11 lands. +- **Phase 37 before any release cut.** release-please exists to publish an artifact that currently cannot start. +- **Phase 37 (PKG-06) before Phase 43 (EXT-03).** If `bundled_plugins_dir()` does not survive a wheel install, the `importlib.resources` fix is Phase 37 work, not workstream H's. +- **Phase 42 (EXT-01) is the first EXT phase.** Every later EXT step multiplies the number of non-conforming plugins reaching paths that currently raise. +- **Phase 46 (EXT-09) before Phase 47 (EXT-11).** The disarm default must already be true when install ships, so no released state has a stranger's freshly installed plugin able to buy by default. +- **Phase 45 (PAR-03) and Phase 46 (EXT-09) are one mechanism.** Both are a pre-transfer gate at `core/orchestrator.py:571`. Phase 45 builds it for the community plugins; adjacent Phase 46 extends it with the third-party disarm default. Two gates that can diverge is the failure mode. +- **EXT-15 (Phase 49) gates milestone completion, not phase ordering.** Phase 50 may follow it. + +Deliberate, non-constraint orderings: + +- Phase 39 (Quality Floor) precedes Phases 41-50 so the milestone's new code is written under an enforced standard rather than retrofitted to one. +- Phase 41's FIX-08 (escape + length-cap the plugin name) precedes Phase 47, which is what makes that value attacker-controlled. +- Phase 40's LICENSE precedes Phase 49's liability language, which lands beside it. +- Phases 42/43/44/46/47/48/49 are H1-H7 from `research/SUMMARY.md` in its reconciled order. That order is researched, not re-derived — do not reshuffle it during planning. + +### Research Flags (v5.0 — carry into planning) + +- **Phase 36:** highest-risk phase in the milestone. Small in requirement count, large in blast radius: a 263-commit merge whose CI has never executed, so the first green run is also the first evidence the merge is correct. Budget verification room, not just merge mechanics. +- **Phase 43:** blocked on PKG-06's factual answer (does the bundled plugin root resolve from an installed wheel?). `plugins*` is in `packages.find` and the plugin files are `.py` modules rather than data files, so discovery probably works — but verify against an actual built wheel before this phase plans. +- **Phase 47:** the consent prompt copy is an acceptance criterion, not an implementation detail. Re-read the consent-fatigue evidence (Böhme and Köpsell CHI 2010; Chrome SSL interstitial clickthrough) before writing it. Type the plugin name back, not `y`; four to six lines of facts, not prose; no bare `--yes`. +- **Phase 47:** two open decisions to make here rather than guess — whether `GITHUB_TOKEN` joins `SECRET_KEYS` or stays environment-only, and whether the install-time import smoke test ships at all (if it does, it must run strictly after consent is granted, never before). +- **Phase 46 or 47:** whether a `sys.addaudithook` detection layer ships, and at what scope. Needs a measurement, not a design opinion. Whatever ships is labelled detection and forensics, never prevention. +- **Phase 49:** where the third-party disclaimer text lives (SECURITY.md section, a new `docs/PLUGIN_TRUST.md`, or both) and whether a first-run acknowledgement exists. The install-time half is Phase 47; the first-run half is an unresolved product call. Liability wording carries a MEDIUM confidence flag and is not legal advice. +- **All EXT phases:** the dominant risk is overclaiming, not a missing feature. Import is execution and nothing in v5.0 changes that. Attach to every shipped control one sentence naming what it does not stop. + +### REVIEW.md Deep-Pass Assignments (v5.0) + +Per RETROSPECTIVE.md lesson 4, two phases carry a post-verification REVIEW.md deep pass: + +- **Phase 45** (Community Plugin Parity + Pre-Transfer Arming Gate) — safety-critical guard. +- **Phase 47** (Fetch, Pre-Flight, Install & Consent) — the milestone's new unauthenticated input surface. Phase 48 extends the same surface; re-run both criteria against `update`/`remove` before closing it. + +Both reviews carry the same two non-negotiable criteria: (a) for every shipped control, one sentence naming what it does not stop; (b) confirmation that no shipped artifact describes third-party plugins as sandboxed, isolated, curated, verified, or safe. EXT-17 makes (b) mechanical. + +### Scope Decision Recorded at Roadmap Time (v5.0) + +- **REL-01 (event-loop stall watchdog) stays in v2.** Research `SUMMARY.md` Gap 5 asked whether it belongs to H, to G, or to a deferred reliability item. It is a reliability control rather than a distribution control, `asyncio.timeout` cannot preempt a blocking plugin regardless of where that plugin came from, and the exposure exists today with the 7 bundled plugins. Folding it into H would silently expand that workstream. Recorded as a decision, not an omission. + +### Key Decisions Logged (v4.1) + +- [Phase 25 — roadmap]: charting library = uPlot 1.6.32 (MIT, ~52KB IIFE + ~1KB CSS, interactive tooltips, Canvas 2D, time series); vendored to `web/static/uplot.min.js` + `web/static/uplot.min.css`; no CDN, no Node. +- [Phase 25 — roadmap]: CSS 3-file split: `tokens.css` (`:root` blocks only), `components.css` (component rules via `var(--xxx)` only), `dashboard.css` (layout + `@import`); each file stays under 200 lines. +- [Phase 25 — roadmap]: FOUC prevention: inline synchronous `/ at 367-368) + - .planning/phases/25-design-system/25-PATTERNS.md (head-block AFTER lines 359-378; header-bar insertion lines 382-392) + - .planning/phases/25-design-system/25-UI-SPEC.md (FOUC Prevention Contract, Header Bar Contract, File Structure Contract head order) + + + Rewrite the `` so the inline FOUC `` at the end of `` (before the existing + inline ` + + + ShopPyBot Dashboard + + + + + +``` + +**Header bar insertion — add before `.container` div** (currently line 17): +```html + +
+ ShopPyBot + + +
+ + + +``` + +**XSS fix — loadItems() current broken code** (line 241): +```javascript +// BEFORE (dashboard.html line 241 — XSS): +tr.innerHTML = `${item.name}${item.link}${item.auto_buy ? 'Yes' : 'No'}${item.quantity}`; + +// AFTER (safe DOM construction): +function makeCell(text) { + const td = document.createElement('td'); + td.textContent = text; + return td; +} + +data.items.forEach(item => { + const tr = document.createElement('tr'); + tr.appendChild(makeCell(item.name)); + tr.appendChild(makeCell(item.link)); + tr.appendChild(makeCell(item.auto_buy ? 'Yes' : 'No')); + tr.appendChild(makeCell(String(item.quantity))); + + const tdAction = document.createElement('td'); + const btn = document.createElement('button'); + btn.className = 'btn-text-destructive'; + btn.type = 'button'; + btn.textContent = 'Remove Item'; + btn.addEventListener('click', () => removeItem(item.link)); + tdAction.appendChild(btn); + tr.appendChild(tdAction); + tbody.appendChild(tr); +}); +``` + +**XSS fix — loadCredentials() current broken code** (lines 255-263): +```javascript +// BEFORE (dashboard.html lines 255-263 — XSS): +div.innerHTML = ` + ${cred.name} + ${cred.is_set ? 'Set' : 'Not set'} +
+ + + +
`; + +// AFTER (safe DOM construction): +(data.credentials || []).forEach(cred => { + const div = document.createElement('div'); + div.className = 'cred-row'; + + const nameSpan = document.createElement('span'); + nameSpan.className = 'cred-name'; + nameSpan.textContent = cred.name; + + const statusSpan = document.createElement('span'); + statusSpan.className = cred.is_set ? 'cred-status' : 'cred-status text-secondary'; + statusSpan.textContent = cred.is_set ? 'Set' : 'Not set'; + + const credForm = document.createElement('div'); + credForm.className = 'cred-form'; + + const input = document.createElement('input'); + input.type = 'password'; + input.placeholder = 'Enter new value'; + input.id = 'cred-input-' + cred.name; + input.autocomplete = 'new-password'; + + const saveBtn = document.createElement('button'); + saveBtn.type = 'button'; + saveBtn.className = 'btn btn-accent btn-sm'; + saveBtn.textContent = 'Save Credential'; + saveBtn.addEventListener('click', () => saveCredential(cred.name)); + + const fb = document.createElement('span'); + fb.className = 'field-feedback'; + fb.id = 'cred-fb-' + cred.name; + + credForm.appendChild(input); + credForm.appendChild(saveBtn); + credForm.appendChild(fb); + div.appendChild(nameSpan); + div.appendChild(statusSpan); + div.appendChild(credForm); + container.appendChild(div); +}); +``` + +**Theme toggle JS — add to ` + + + + +``` + +No `data-theme` is set if the user has no preference and OS prefers light. The +`:root` block is the light default -- attribute absence = light mode. + +### Pattern 3: Safe DOM Construction (XSS Fix) + +**What:** Replace `innerHTML = template_literal` with `createElement` + `textContent` +assignment for all API-sourced values. + +**When to use:** Everywhere API data enters the DOM. Not needed for fully static +string literals with no API values interpolated. + +**Example -- loadItems() fix:** +```javascript +// BEFORE (broken -- XSS): +tr.innerHTML = `${item.name}${item.link}...`; + +// AFTER (safe): +function makeCell(text) { + const td = document.createElement('td'); + td.textContent = text; + return td; +} + +data.items.forEach(item => { + const tr = document.createElement('tr'); + tr.dataset.link = item.link; + tr.appendChild(makeCell(item.name)); + tr.appendChild(makeCell(item.link)); + tr.appendChild(makeCell(item.auto_buy ? 'Yes' : 'No')); + tr.appendChild(makeCell(String(item.quantity))); + + const tdAction = document.createElement('td'); + const btn = document.createElement('button'); + btn.className = 'btn-text-destructive'; + btn.type = 'button'; + btn.textContent = 'Remove Item'; + btn.addEventListener('click', () => removeItem(item.link)); + tdAction.appendChild(btn); + tr.appendChild(tdAction); + + tbody.appendChild(tr); +}); +``` + +**escHtml helper** (for unavoidable SVG string interpolation only -- not needed in Phase 25): +```javascript +function escHtml(str) { + const d = document.createElement('div'); + d.textContent = str; + return d.innerHTML; +} +``` + +### Pattern 4: Theme Toggle Wiring + +**What:** Button in sticky header reads current `data-theme`, toggles it, persists +to `localStorage`, and updates `aria-label`. + +```javascript +const toggleBtn = document.getElementById('theme-toggle'); + +toggleBtn.addEventListener('click', () => { + const current = document.documentElement.dataset.theme || 'light'; + const next = current === 'dark' ? 'light' : 'dark'; + document.documentElement.dataset.theme = next; + localStorage.setItem('theme', next); + toggleBtn.setAttribute('aria-label', next === 'dark' ? 'Switch to light mode' : 'Switch to dark mode'); + toggleBtn.textContent = next === 'dark' ? '☾' : '☀'; // ☾ or ☀ +}); +``` + +### Anti-Patterns to Avoid + +- **`@import url('https://...')`:** Any external URL in CSS breaks the zero-CDN constraint and will fail security review. +- **`innerHTML` with template literals containing API data:** The exact bug being fixed. Post-fix, the CI grep assertion must block regression. +- **`onclick="funcName(value)"` attribute on dynamically created elements:** Injects values into an HTML attribute string; replace with `addEventListener`. +- **Hardcoded hex in `components.css`:** Every color must be a `var(--xxx)` call. Hardcoded hex breaks dark mode. +- **FOUC script as external file (` + + + + +``` + +### Header Bar HTML +```html + +
+ ShopPyBot + + +
+``` + +### CI Grep Assertion for XSS Regression (pytest) +```python +# Source: 25-UI-SPEC.md XSS Fix Contract -- CI assertion +import re, pathlib + +def test_no_innerHTML_with_api_data(): + """Block innerHTML assignments that interpolate API-sourced variables.""" + html = pathlib.Path("web/templates/dashboard.html").read_text(encoding="utf-8") + pattern = re.compile(r'innerHTML\s*=\s*.*\$\{(item\.|cred\.|data\.|cfg\.|resp\.)') + matches = pattern.findall(html) + assert not matches, f"innerHTML with API data found: {matches}" +``` + +### tokens.css Skeleton (key color tokens) +```css +/* Source: 25-UI-SPEC.md Color Tokens sections */ +:root { + /* spacing */ + --space-xs: 4px; --space-sm: 8px; --space-md: 16px; + --space-lg: 24px; --space-xl: 32px; --space-2xl: 48px; + /* typography */ + --text-sm: 12px; --text-body: 14px; --text-lg: 18px; --text-xl: 22px; + --weight-normal: 400; --weight-semibold: 600; + --leading-tight: 1.2; --leading-body: 1.5; --leading-ui: 1.4; + --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + /* light colors */ + --color-bg: #f5f5f5; + --color-surface: #ffffff; + --color-border: #d1d5db; + --color-text: #111827; + --color-text-muted: #6b7280; + --color-accent: #2563eb; + --color-accent-fg: #ffffff; + --color-destructive: #dc2626; + --color-destructive-fg: #ffffff; + --color-destructive-subtle: #fee2e2; + --color-status-ok: #16a34a; + --color-status-neutral: #6b7280; + --color-status-warn: #d97706; + --color-status-err: #dc2626; +} + +[data-theme="dark"] { + --color-bg: #0f172a; + --color-surface: #1e293b; + --color-border: #334155; + --color-text: #f1f5f9; + --color-text-muted: #94a3b8; + --color-accent: #3b82f6; + --color-accent-fg: #ffffff; + --color-destructive: #ef4444; + --color-destructive-fg: #ffffff; + --color-destructive-subtle: #450a0a; + --color-status-ok: #22c55e; + --color-status-neutral: #64748b; + --color-status-warn: #f59e0b; + --color-status-err: #ef4444; +} +``` + +## State of the Art + +| Old Approach | Current Approach | When Changed | Impact | +|--------------|------------------|--------------|--------| +| Single `dashboard.css` with hardcoded hex | 3-file split: tokens + components + layout | Phase 25 | Enables dark mode without any JS; tokens drive all color decisions | +| `innerHTML` template string for DOM building | `createElement` + `textContent` | Phase 25 | Closes XSS vector; `bold` in item name renders as literal text | +| No theme support (light only) | `data-theme` attribute + `prefers-color-scheme` auto | Phase 25 | Operator comfort for night-time use; persisted preference survives reload | +| No sticky header | `
` sticky top | Phase 25 | Establishes slot for OBS-09 uptime (Phase 28) and SSE Live indicator (Phase 29) | + +**Deprecated/outdated in this codebase after Phase 25:** +- Hardcoded hex in CSS: replaced by token variables +- `onclick="funcName(value)"` attribute pattern on dynamic elements: replaced by `addEventListener` + +## Assumptions Log + +| # | Claim | Section | Risk if Wrong | +|---|-------|---------|---------------| +| A1 | `StaticFiles` mount at `/static/` serves subdirectories including `vendor/` without additional configuration | Pitfall 2, Architecture | If wrong, uPlot files at `/static/vendor/` return 404; fix is to move files to `/static/` root or update StaticFiles mount path | +| A2 | The FOUC inline script as FIRST child of `` (before even ``) is valid HTML and executes synchronously in all target browsers | Architecture, Pitfall 1 | If a browser defers the script, FOUC occurs; risk is LOW -- this is a well-documented pattern | +| A3 | uPlot 1.6.32 IIFE file is named `uPlot.iife.min.js` (uppercase P) on the GitHub release; template reference uses lowercase `uplot.iife.min.js` | Standard Stack | Case mismatch causes 404 on case-sensitive filesystems (Linux); Windows is case-insensitive so may not surface in dev; commit with consistent lowercase name | + +## Open Questions + +1. **uPlot filename casing** + - What we know: GitHub dist directory lists `uPlot.iife.min.js` (capital P). UI-SPEC.md references `uplot.iife.min.js` (lowercase). [VERIFIED: github.com/leeoniya/uPlot/tree/1.6.32/dist] + - What's unclear: The deploy target (Windows dev) is case-insensitive, but if this repo ever runs on Linux, a case mismatch causes 404. + - Recommendation: Commit the file as `uplot.iife.min.js` (all lowercase, matching the template ` + + + + ... + +``` + +Vendor assets: + +| File | Contents | Served from | +|------|----------|-------------| +| `web/static/vendor/uplot.iife.min.js` | uPlot 1.6.32 IIFE build (~52 KB) | `/static/vendor/uplot.iife.min.js` | +| `web/static/vendor/uplot.min.css` | uPlot companion CSS (~1 KB) | `/static/vendor/uplot.min.css` | + +uPlot is loaded via `` and `` in the template. No CDN reference in any file. + +--- + +## Spacing Scale + +Declared values (all multiples of 4). Promoted to CSS custom properties in `tokens.css` `:root`. + +| Token | CSS Custom Property | Value | Usage | +|-------|---------------------|-------|-------| +| xs | `--space-xs` | 4px | Icon gaps, inline padding, form-group internal gap | +| sm | `--space-sm` | 8px | Button padding vertical, table cell padding, log-pre padding, button-row gap | +| md | `--space-md` | 16px | Card padding horizontal, form-group margin-bottom, section spacing, container horizontal padding | +| lg | `--space-lg` | 24px | Card padding, cred-row margin-bottom | +| xl | `--space-xl` | 32px | Container top/bottom padding, h1 margin-bottom | +| 2xl | `--space-2xl` | 48px | Card margin-bottom between sections | +| 3xl | `--space-3xl` | 64px | Not used in Phase 25; reserved for future page-level spacing | + +Exceptions: `.btn` min-height is 36px (not on scale — inherited from existing, preserved for clickability). `.btn-sm` min-height is 30px (inherited). `.status-dot` is 10px (decorative indicator, not spacing). These are dimension values, not spacing, so no scale exception is needed. + +--- + +## Typography + +Exact values. Base size is 14px (CONTEXT.md: "14px base"). System font stack (no external fonts). + +| Role | Size | Weight | Line Height | CSS Custom Properties | +|------|------|--------|-------------|----------------------| +| Body | 14px | 400 | 1.5 | `--text-body`, `--weight-normal: 400` | +| Label / UI text | 14px | 600 | 1.4 | `--weight-semibold: 600` | +| Code / Log | 12px | 400 | 1.4 | `--text-sm` | +| Section heading (h2) | 18px | 600 | 1.2 | `--text-lg` | +| Page heading (h1) | 22px | 600 | 1.2 | `--text-xl` | + +Only two font weights: 400 (normal) and 600 (semibold). No other weights. +Only four font sizes: 12px, 14px, 18px, 22px. + +Typography custom properties in `tokens.css`: + +```css +:root { + --text-sm: 12px; + --text-body: 14px; + --text-lg: 18px; + --text-xl: 22px; + --weight-normal: 400; + --weight-semibold: 600; + --leading-tight: 1.2; + --leading-body: 1.5; + --leading-ui: 1.4; + --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; +} +``` + +--- + +## Color Tokens — Light Mode + +60% dominant (page background), 30% secondary (cards, inputs, panels), 10% accent (reserved elements listed below). + +| Token | CSS Custom Property | Light Value | Semantic Role | +|-------|---------------------|-------------|---------------| +| Page background | `--color-bg` | `#f5f5f5` | Body background (60%) | +| Card / surface | `--color-surface` | `#ffffff` | Cards, form inputs, log panel (30%) | +| Border | `--color-border` | `#d1d5db` | Card borders, input borders, table row dividers | +| Text primary | `--color-text` | `#111827` | Body text, labels, table cell text | +| Text secondary | `--color-text-muted` | `#6b7280` | Placeholder text, `.text-secondary`, empty-state copy | +| Accent | `--color-accent` | `#2563eb` | Reserved elements listed below (10%) | +| Accent foreground | `--color-accent-fg` | `#ffffff` | Text on accent backgrounds | +| Destructive | `--color-destructive` | `#dc2626` | Destructive actions only (listed below) | +| Destructive foreground | `--color-destructive-fg` | `#ffffff` | Text on destructive backgrounds | +| Destructive subtle bg | `--color-destructive-subtle` | `#fee2e2` | Banner warning background | +| Status: running | `--color-status-ok` | `#16a34a` | Running status dot, success feedback | +| Status: stopped | `--color-status-neutral` | `#6b7280` | Stopped status dot | +| Status: warning | `--color-status-warn` | `#d97706` | Amber staleness band (30-60s), reserved for Phase 28 | +| Status: error | `--color-status-err` | `#dc2626` | Red staleness band (>60s), error feedback | + +--- + +## Color Tokens — Dark Mode + +Declared in `tokens.css` under `[data-theme="dark"]` block. Values override the `:root` light defaults. + +| Token | CSS Custom Property | Dark Value | Notes | +|-------|---------------------|------------|-------| +| Page background | `--color-bg` | `#0f172a` | Dark slate page background | +| Card / surface | `--color-surface` | `#1e293b` | Card and input surface | +| Border | `--color-border` | `#334155` | Subtle border on dark | +| Text primary | `--color-text` | `#f1f5f9` | High-contrast body text | +| Text secondary | `--color-text-muted` | `#94a3b8` | Muted text on dark | +| Accent | `--color-accent` | `#3b82f6` | Blue-500 for dark (lighter than light-mode #2563eb for contrast) | +| Accent foreground | `--color-accent-fg` | `#ffffff` | Text on accent | +| Destructive | `--color-destructive` | `#ef4444` | Red-500 on dark | +| Destructive foreground | `--color-destructive-fg` | `#ffffff` | Text on destructive | +| Destructive subtle bg | `--color-destructive-subtle` | `#450a0a` | Banner warning bg on dark | +| Status: running | `--color-status-ok` | `#22c55e` | Green-500 for dark | +| Status: stopped | `--color-status-neutral` | `#64748b` | Slate-500 for dark | +| Status: warning | `--color-status-warn` | `#f59e0b` | Amber-400 for dark | +| Status: error | `--color-status-err` | `#ef4444` | Red-500 for dark | + +Accent reserved for (Phase 25 scope only): +- `.btn-accent` background (primary action buttons: "Start Bot", "Add Item", "Save Credential", "Save Config") +- Input focus ring (`:focus-visible` outline) +- Theme toggle button active ring + +Destructive reserved for (Phase 25 scope only): +- `.btn-destructive` background ("Stop Bot") +- `.btn-text-destructive` text color ("Remove Item" buttons) +- `.field-error` text color (inline form validation messages) +- `.field-feedback.error` text color +- `.banner-warning` text and border (uses `--color-destructive`) + +--- + +## FOUC Prevention Contract + +The inline script is the FIRST child of ``, placed before any `` element. It runs synchronously before the browser requests any CSS. It must be inlined directly in the template — not a `src=` external file. + +```html + + + ...CSS links... + +``` + +If no stored preference and no `prefers-color-scheme` match, no `data-theme` attribute is set. The `:root` block (light) is the default. This means the attribute is absent in light-pref browsers, which is fine — light is the base `:root`. + +--- + +## Theme Toggle Contract + +| Property | Spec | +|----------|------| +| Placement | Sticky top header bar, right-aligned | +| Element | ` + + +
+ +