diff --git a/.github/scripts/check-deploy-version.mjs b/.github/scripts/check-deploy-version.mjs new file mode 100644 index 0000000..e441d74 --- /dev/null +++ b/.github/scripts/check-deploy-version.mjs @@ -0,0 +1,421 @@ +#!/usr/bin/env node +/** + * Deploy verdict gate — did this deploy actually publish a new Worker version? + * + * ## Why the step's exit code is not the answer + * + * Between 2026-08-25 and 2026-09-04 this repository ran 36 deploys that + * published nothing. Cloudflare refuses an oversized upload at version + * creation, so no version is created and the previously accepted one keeps + * serving — the site stays up, stale, and the difference between "deployed" + * and "rejected" is not visible from outside the step. #269 asks for the one + * reading that distinguishes them: a NEW version id, serving, afterwards. + * + * So this gate reads three things rather than one: + * + * --before `wrangler deployments status --json`, taken BEFORE the deploy + * --deploy-log everything the deploy command printed + * --after the same status reading, taken AFTER + * + * and requires all of: the command succeeded, it named exactly one new version + * id, that id is well-formed, it is not the id that was already serving, and + * the post-deploy reading agrees that it is what is serving now. + * + * ## Absent readings are failures, never silence + * + * Every "could not read that" path here is a finding, not a skip. That is the + * whole point: a check that quietly passes when its input is missing is the + * shape this lane has been bitten by three times in one day, and it is exactly + * what an unreadable `--before` would produce — no previous id to compare + * against, therefore nothing to disagree with, therefore green. + * + * `--allow-no-previous` exists for the one legitimate case, a Worker with no + * deployments at all, and it has to be asked for explicitly. + * + * ## Usage + * + * node .github/scripts/check-deploy-version.mjs \ + * --before before.json --after after.json \ + * --deploy-log deploy.log --deploy-exit 0 + * + * node .github/scripts/check-deploy-version.mjs --self-test + * + * When `GITHUB_OUTPUT` is set, `previous_version_id` and `new_version_id` are + * written to it before any exit, so a rollback job downstream has the id to + * roll back to even on the runs where this gate goes red. + */ + +import { readFileSync, existsSync, appendFileSync } from 'node:fs'; + +/** Every rule this gate enforces; the self-test asserts each has a red fixture. */ +const RULES = [ + 'deploy-command-failed', + 'no-version-id', + 'multiple-version-ids', + 'malformed-version-id', + 'unreadable-previous', + 'unchanged-version', + 'unreadable-after', + 'not-serving', +]; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * The line `wrangler deploy` prints on success, and the only place a version id + * is taken from. Anchored to the line so a version id quoted inside an error + * message cannot be read as a successful publish. + */ +const VERSION_LINE = /^[^\S\n]*Current Version ID:[^\S\n]*(\S+)[^\S\n]*$/gim; + +/** + * Pull the serving version id out of a `wrangler deployments status --json` + * capture. + * + * Returns `{ id }`, `{ empty: true }` when the capture is blank (the command + * produced no stdout — typically because it failed), or `{ error }` when there + * is text that does not yield an id. Those three are deliberately different + * answers: only the middle one can be waived. + */ +function servingVersion(text) { + const trimmed = (text ?? '').trim(); + if (!trimmed) return { empty: true }; + + let parsed; + try { + parsed = JSON.parse(trimmed); + } catch { + // Wrangler prints nothing but the document under `--json`, but a proxy + // banner or a trailing notice would still leave one object in the stream. + const start = trimmed.indexOf('{'); + const end = trimmed.lastIndexOf('}'); + if (start === -1 || end <= start) return { error: 'not JSON and no object found' }; + try { + parsed = JSON.parse(trimmed.slice(start, end + 1)); + } catch (e) { + return { error: `unparseable JSON: ${e.message}` }; + } + } + + const versions = Array.isArray(parsed?.versions) ? parsed.versions : null; + if (!versions || versions.length === 0) return { error: 'JSON carries no versions[]' }; + + const full = versions.find((v) => Number(v?.percentage) === 100); + const chosen = full ?? versions[0]; + const id = chosen?.version_id; + if (typeof id !== 'string' || !UUID.test(id)) { + return { error: `versions[0].version_id is not a UUID: ${JSON.stringify(id)}` }; + } + return { id, split: versions.length > 1 && !full }; +} + +/** Every distinct version id the deploy command claimed to have published. */ +function publishedVersions(log) { + return [...new Set([...(log ?? '').matchAll(VERSION_LINE)].map((m) => m[1]))]; +} + +/** + * Judge one deploy. Pure — the self-test drives it with fixture strings. + * + * `inputs` is `{ before, after, deployLog, deployExit, allowNoPrevious }`, + * where the three text fields are file CONTENTS, not paths. + */ +function evaluate(inputs) { + const findings = []; + const add = (rule, detail) => findings.push({ rule, detail }); + + const previous = servingVersion(inputs.before); + const after = inputs.after === null || inputs.after === undefined ? null : servingVersion(inputs.after); + const published = publishedVersions(inputs.deployLog); + + const measured = { + previousVersionId: previous.id ?? null, + newVersionId: null, + servingAfter: after?.id ?? null, + deployExit: inputs.deployExit, + }; + + if (previous.error) { + add('unreadable-previous', `pre-deploy status: ${previous.error}`); + } else if (previous.empty && !inputs.allowNoPrevious) { + add( + 'unreadable-previous', + 'pre-deploy status was empty — without it "a new version is serving" cannot be ' + + 'asserted at all (pass --allow-no-previous only for a Worker that has never deployed)', + ); + } + + if (Number(inputs.deployExit) !== 0) { + add('deploy-command-failed', `the deploy command exited ${inputs.deployExit}`); + } + + if (published.length === 0) { + add( + 'no-version-id', + 'the deploy printed no "Current Version ID:" line — nothing was published, ' + + 'whatever the command exited with', + ); + } else if (published.length > 1) { + add('multiple-version-ids', `the deploy named ${published.length} version ids: ${published.join(', ')}`); + } else if (!UUID.test(published[0])) { + add('malformed-version-id', `"Current Version ID: ${published[0]}" is not a UUID`); + } else { + measured.newVersionId = published[0]; + } + + if (measured.newVersionId && previous.id && measured.newVersionId === previous.id) { + add( + 'unchanged-version', + `${measured.newVersionId} was already serving before this deploy — the upload ` + + 'created no new version', + ); + } + + if (after) { + if (after.error || after.empty) { + add('unreadable-after', `post-deploy status: ${after.error ?? 'empty capture'}`); + } else if (measured.newVersionId && after.id !== measured.newVersionId) { + add( + 'not-serving', + `the deploy published ${measured.newVersionId} but ${after.id} is serving afterwards`, + ); + } + } + + return { findings, measured }; +} + +/* ------------------------------------------------------------------ main -- */ + +function readIfPresent(path) { + if (!path) return null; + return existsSync(path) ? readFileSync(path, 'utf8') : ''; +} + +function emitOutputs(measured) { + const out = process.env.GITHUB_OUTPUT; + if (!out) return; + appendFileSync( + out, + `previous_version_id=${measured.previousVersionId ?? ''}\n` + + `new_version_id=${measured.newVersionId ?? ''}\n`, + ); +} + +function gate(opts) { + const { findings, measured } = evaluate({ + before: readIfPresent(opts.before), + after: opts.after ? readIfPresent(opts.after) : null, + deployLog: readIfPresent(opts.deployLog), + deployExit: opts.deployExit, + allowNoPrevious: opts.allowNoPrevious, + }); + + // Written before any exit: the rollback job needs the previous id most + // precisely on the runs where this gate is about to go red. + emitOutputs(measured); + + console.log('deploy verdict'); + console.log(` serving before : ${measured.previousVersionId ?? '(unreadable)'}`); + console.log(` published now : ${measured.newVersionId ?? '(none)'}`); + console.log(` serving after : ${measured.servingAfter ?? '(not read)'}`); + console.log(` command exit : ${measured.deployExit}`); + console.log(''); + + if (findings.length) { + for (const f of findings) console.error(` [${f.rule}] ${f.detail}`); + console.error(`\n✗ deploy: ${findings.length} finding(s) — this run published nothing verifiable`); + process.exitCode = 1; + return; + } + console.log(`✓ deploy: ${measured.newVersionId} is serving, replacing ${measured.previousVersionId}`); +} + +/* ------------------------------------------------------------- self-test -- */ + +const A = '69c79ee3-1f2b-4c6d-8a90-0b1c2d3e4f50'; +const B = '7ad81ff4-2039-4d7e-9ba1-1c2d3e4f5061'; + +const status = (id, extra = {}) => + JSON.stringify({ + created_on: '2026-08-25T15:38:00.000Z', + author_email: 'ci@objectos.ai', + versions: [{ version_id: id, percentage: 100 }], + ...extra, + }); + +const deployLog = (id) => + ['Total Upload: 58541.02 KiB / gzip: 12002.11 KiB', 'Uploaded docs-objectos (24.31 sec)', `Current Version ID: ${id}`, ''].join('\n'); + +const REJECTED_LOG = [ + 'Total Upload: 66112.94 KiB / gzip: 13991.02 KiB', + '', + 'X [ERROR] A request to the Cloudflare API (/accounts/…/workers/scripts/docs-objectos/versions) failed.', + '', + ' Script startup exceeded CPU time limit. [code: 10027]', + '', +].join('\n'); + +const CASES = [ + { + name: 'a real deploy trips nothing', + inputs: { before: status(A), after: status(B), deployLog: deployLog(B), deployExit: 0 }, + expect: [], + }, + { + name: 'rejected upload, non-zero exit', + inputs: { before: status(A), after: status(A), deployLog: REJECTED_LOG, deployExit: 1 }, + expect: ['deploy-command-failed', 'no-version-id'], + }, + { + name: 'rejected upload that exits 0 anyway', + inputs: { before: status(A), after: status(A), deployLog: REJECTED_LOG, deployExit: 0 }, + expect: ['no-version-id'], + }, + { + name: 'two version ids in one log', + inputs: { + before: status(A), + after: status(B), + deployLog: `${deployLog(B)}\n${deployLog(A)}`, + deployExit: 0, + }, + expect: ['multiple-version-ids'], + }, + { + name: 'version id is not a uuid', + inputs: { before: status(A), after: status(B), deployLog: 'Current Version ID: undefined\n', deployExit: 0 }, + expect: ['malformed-version-id'], + }, + { + name: 'pre-deploy status empty', + inputs: { before: '', after: status(B), deployLog: deployLog(B), deployExit: 0 }, + expect: ['unreadable-previous'], + }, + { + name: 'pre-deploy status empty, waived', + inputs: { + before: '', + after: status(B), + deployLog: deployLog(B), + deployExit: 0, + allowNoPrevious: true, + }, + expect: [], + }, + { + name: 'pre-deploy status is an error page', + inputs: { + before: 'Authentication error [code: 10000]', + after: status(B), + deployLog: deployLog(B), + deployExit: 0, + }, + expect: ['unreadable-previous'], + }, + { + name: 'the same version "published" twice', + inputs: { before: status(A), after: status(A), deployLog: deployLog(A), deployExit: 0 }, + expect: ['unchanged-version'], + }, + { + name: 'post-deploy status unreadable', + inputs: { before: status(A), after: '{"versions":[]}', deployLog: deployLog(B), deployExit: 0 }, + expect: ['unreadable-after'], + }, + { + name: 'published, but something else is serving', + inputs: { before: status(A), after: status(A), deployLog: deployLog(B), deployExit: 0 }, + expect: ['not-serving'], + }, +]; + +/** Shapes `servingVersion` must read, or must refuse to read. */ +const STATUS_CASES = [ + ['plain json', status(A), A], + ['json behind a banner', `⛅️ wrangler 4.95.0\n${status(A)}`, A], + ['gradual rollout takes the first entry', JSON.stringify({ versions: [{ version_id: A, percentage: 60 }, { version_id: B, percentage: 40 }] }), A], + ['100% entry wins over order', JSON.stringify({ versions: [{ version_id: A, percentage: 0 }, { version_id: B, percentage: 100 }] }), B], + ['no versions array', '{"created_on":"x"}', null], + ['not json at all', 'The Worker docs-objectos has no deployments.', null], + ['blank', ' \n', null], +]; + +function selfTest() { + let failed = 0; + + for (const c of CASES) { + const { findings } = evaluate({ after: null, ...c.inputs }); + const fired = [...new Set(findings.map((f) => f.rule))].sort(); + const want = [...c.expect].sort(); + const ok = fired.join(',') === want.join(','); + if (!ok) failed += 1; + console.log( + `${ok ? '✓' : '✗'} ${c.name.padEnd(40)} fired [${fired.join(' ') || '—'}]` + + (ok ? '' : ` expected [${want.join(' ') || '—'}]`), + ); + if (!ok) for (const f of findings) console.error(` [${f.rule}] ${f.detail}`); + } + + console.log(''); + for (const [name, text, expected] of STATUS_CASES) { + const got = servingVersion(text); + const ok = (got.id ?? null) === expected; + if (!ok) failed += 1; + console.log( + `${ok ? '✓' : '✗'} status ${name.padEnd(34)} -> ${got.id ?? (got.empty ? '(empty)' : `(refused: ${got.error})`)}`, + ); + } + + console.log(''); + const covered = new Set(CASES.flatMap((c) => c.expect)); + for (const rule of RULES) { + if (!covered.has(rule)) { + console.error(`✗ rule "${rule}" has no fixture that trips it`); + failed += 1; + } + } + if (!CASES.some((c) => c.expect.length === 0)) { + console.error('✗ no fixture asserts that a good deploy trips nothing'); + failed += 1; + } + + if (failed) { + console.error(`\n✗ self-test: ${failed} case(s) did not behave as declared`); + process.exitCode = 1; + return; + } + console.log( + `✓ self-test: ${CASES.length} deploy case(s) and ${STATUS_CASES.length} status-parse case(s) — ` + + `all ${RULES.length} rules demonstrated able to fail`, + ); +} + +function parseArgs(argv) { + const opts = { before: null, after: null, deployLog: null, deployExit: '0', allowNoPrevious: false }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + if (a === '--before') opts.before = argv[++i]; + else if (a === '--after') opts.after = argv[++i]; + else if (a === '--deploy-log') opts.deployLog = argv[++i]; + else if (a === '--deploy-exit') opts.deployExit = argv[++i]; + else if (a === '--allow-no-previous') opts.allowNoPrevious = true; + else { + console.error(`unknown argument: ${a}`); + process.exit(2); + } + } + if (!opts.before || !opts.deployLog) { + console.error('✗ --before and --deploy-log are required (an absent reading is a failure, not a skip)'); + process.exit(2); + } + return opts; +} + +function main() { + const argv = process.argv.slice(2); + if (argv.some((a) => a === '--self-test')) return selfTest(); + return gate(parseArgs(argv)); +} + +main(); diff --git a/.github/scripts/smoke-docs.mjs b/.github/scripts/smoke-docs.mjs new file mode 100644 index 0000000..4535942 --- /dev/null +++ b/.github/scripts/smoke-docs.mjs @@ -0,0 +1,667 @@ +#!/usr/bin/env node +/** + * Live-site smoke check for the published docs Worker. + * + * ## Why this exists + * + * Nothing in this repository's CI ever touched the website. `build`, the Node + * floor, the locale surface, the translation gates and the half-state sweeper + * all judge the tree; the deploy published a Worker and no check anywhere asked + * whether the result rendered. On 2026-09-04 a deploy was accepted, the site + * broke, and a human found it (#261). This script is the check that was + * missing, and `ci.yml` runs it after every deploy (#269). + * + * ## A 200 is not the assertion + * + * The failure this exists to catch returns 200. A Next.js app whose render path + * is broken still serves a document — a shell with the scripts, the styles and + * no content. So status is one rule out of eleven, and the ones that carry the + * weight are structural: + * + * - an `h1` element with text, matching what that page is supposed to be + * called; + * - visible text above a floor, measured after `script` and `style` are + * removed. An empty shell is mostly script: it is large in bytes and close + * to empty in prose, which is why a byte floor alone would pass it; + * - a minimum number of same-site links, which is the docs sidebar. A shell + * has none; + * - the document language, and the path the request finally landed on after + * redirects — `/en/docs` must normalise to `/docs`, and a redirect loop or + * a 404-to-negotiation loop shows up here rather than as a 200 somewhere + * unexpected. + * + * ## The negative control is part of every run + * + * This lane has had three same-day instances of a probe that could not fail: + * a port check reading a command that prints nothing in that container, an + * ablation that returned 500 because the server never started, and a scan whose + * script exited before it scanned. A green from this script means nothing + * unless the same code path is able to produce a red against the same host in + * the same run. + * + * So every run also fetches a path that must NOT render — `/docs/` plus a slug + * no page claims — and requires it to produce findings. If the negative control + * comes back clean, `negative-control-passed` fires and the run fails: whatever + * that means, it means this check is not currently able to tell a rendered page + * from an unrendered one, and its green is worthless. + * + * That is a live demonstration, not a fixture one, and it costs one request. + * `--self-test` is the offline half: every rule below has a fixture that trips + * it, and the runner asserts that the set of rules with a red fixture is the + * whole set. Weaken a rule and the self-test exits 1. + * + * ## Usage + * + * node .github/scripts/smoke-docs.mjs # default targets + * node .github/scripts/smoke-docs.mjs --base https://host # another origin + * node .github/scripts/smoke-docs.mjs --paths /a,/b # ad-hoc targets + * node .github/scripts/smoke-docs.mjs --self-test # prove the rules + * + * Exit 0 only when every target passed every rule AND the negative control + * failed at least one. Any other outcome exits 1. + * + * ## What the expectations may and may not assume + * + * The expectations below are deliberately generic — an `h1` noun, a language, + * a normalised path — and never a sentence out of a page body. This script has + * to be runnable against whatever version is currently serving, which is not + * necessarily built from the commit it runs on: for the whole of 2026-08-25 to + * 2026-09-04 the live site was pinned to a version 36 rejected deploys older + * than `main`. An expectation derived from the working tree would have been a + * gate on content drift wearing a smoke check's name. + */ + +/** The origin the checks run against unless `--base` says otherwise. */ +const DEFAULT_BASE = 'https://docs.objectos.ai'; + +/** + * Every rule this script enforces. The self-test asserts each one has a fixture + * that makes it fire; a rule added here without a fixture fails the self-test, + * which is what keeps this list from growing decorative entries. + */ +const RULES = [ + 'fetch-failed', + 'status', + 'not-html', + 'error-shell', + 'too-little-text', + 'no-title', + 'no-h1', + 'h1-mismatch', + 'few-links', + 'lang-mismatch', + 'final-path', + 'negative-control-passed', +]; + +/** + * Strings that mean the response is an error surface rather than a page. + * + * Matched against the document title and the first heading only, never the + * whole body: `content/docs/` is documentation about running a server, and a + * page that explains what a 500 means must not be read as one. None of these + * occurs in the corpus today, and scoping the match to the title and heading is + * what keeps that from becoming a maintenance trap. + */ +const ERROR_MARKERS = [ + 'this page could not be found', + 'internal server error', + 'application error', + 'worker threw exception', + 'error 1101', + 'error 1102', + 'exceeded its cpu time limit', + '502 bad gateway', + '503 service temporarily unavailable', +]; + +/** + * Defaults every target inherits unless it overrides them. + * + * The two floors are set against MEASURED values, in both directions, because + * a false red here now dispatches a rollback. Taken from the live site on + * 2026-09-04: the four targets carried 4639 to 9101 visible characters and 14 + * to 22 same-site links; the 404 shell carried 8 characters and 0 links in + * 37962 bytes. So the floors sit an order of magnitude below the smallest real + * page and far above the shell — and that shell is also why the text floor is + * measured on prose rather than bytes. + */ +const TARGET_DEFAULTS = { + status: 200, + lang: 'en', + minText: 500, + minLinks: 8, +}; + +/** + * The pages this checks, and why each one is here. + * + * `/` and `/en/docs` are both required by #269 and both are also redirect + * assertions: `/` is rewritten to `/en` by `middleware.ts` and then redirected + * to `/docs` by `app/[lang]/page.tsx`, and `/en/docs` is 307'd to `/docs` by + * the default-locale strip. Landing anywhere else means the locale routing is + * broken even if a page rendered. + * + * The deep page is three segments down and behind a `next.config.mjs` redirect + * table, so it exercises the part of the route tree a shallow check misses. + * Both content pages predate the version currently serving (added 2026-05-24 + * and earlier), so this list is runnable against the pinned live version. + */ +const TARGETS = [ + { path: '/', finalPath: '/docs', h1: /^ObjectOS$/i }, + { path: '/en/docs', finalPath: '/docs', h1: /^ObjectOS$/i }, + { path: '/docs/quickstart', finalPath: '/docs/quickstart', h1: /^Quickstart$/i }, + { + path: '/docs/build/interface/views', + finalPath: '/docs/build/interface/views', + h1: /^Views$/i, + }, +]; + +/** + * The path the negative control asks for. + * + * `dynamicParams = false` on the docs route rejects an unknown slug at the + * routing level, so this is answered by the root `_not-found` entry: 404, an + * `h1` reading "404", almost no prose and no sidebar. Several rules therefore + * fire on it, which is the point — a live red from the same code path that + * produced the greens. + */ +const NEGATIVE_CONTROL_PATH = '/docs/objectos-smoke-negative-control-269'; + +/* ------------------------------------------------------------- extraction -- */ + +const stripTags = (html) => html.replace(/<[^>]*>/g, ''); + +const decodeEntities = (text) => + text + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/�*39;/g, "'") + .replace(/�*27;/gi, "'"); + +const collapse = (text) => decodeEntities(stripTags(text)).replace(/\s+/g, ' ').trim(); + +/** The document's declared language, or null when there is no `html` element. */ +function htmlLang(body) { + const m = /]*\slang=["']([^"']*)["']/i.exec(body); + return m ? m[1].trim() : null; +} + +/** The `title` element's text, or null. */ +function titleText(body) { + const m = /]*>([\s\S]*?)<\/title>/i.exec(body); + return m ? collapse(m[1]) : null; +} + +/** The FIRST `h1` element's text, or null. */ +function h1Text(body) { + const m = /]*>([\s\S]*?)<\/h1>/i.exec(body); + return m ? collapse(m[1]) : null; +} + +/** + * Visible prose length, with `script` and `style` bodies removed first. + * + * This is the measurement a byte floor cannot make. A broken Next.js render + * still ships every bundle in the document, so an empty shell is tens of + * kilobytes of `script` and a handful of visible characters. + */ +function visibleText(body) { + const withoutCode = body + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(//g, ' '); + return collapse(withoutCode); +} + +/** Distinct same-site link targets in the document — in practice, the sidebar. */ +function internalLinks(body) { + const found = new Set(); + for (const m of body.matchAll(/]*\shref=["'](\/[^"'#?]*)["']/gi)) found.add(m[1]); + return found; +} + +/* ----------------------------------------------------------------- rules -- */ + +/** + * Judge one response against one target. Pure: takes a already-fetched + * response shape, returns findings and the measurements behind them. + * + * The response shape is `{ error, status, url, contentType, body }` so the + * self-test can drive every rule without a network. + */ +function evaluate(target, res) { + const spec = { ...TARGET_DEFAULTS, ...target }; + const findings = []; + const add = (rule, detail) => findings.push({ rule, detail }); + + if (res.error) { + add('fetch-failed', `${spec.path}: ${res.error}`); + return { findings, measured: { error: res.error } }; + } + + const body = res.body ?? ''; + const measured = { + status: res.status, + finalPath: res.url ? new URL(res.url).pathname : null, + contentType: res.contentType ?? null, + bytes: Buffer.byteLength(body, 'utf8'), + lang: htmlLang(body), + title: titleText(body), + h1: h1Text(body), + text: visibleText(body).length, + links: internalLinks(body).size, + }; + + if (measured.status !== spec.status) { + add('status', `${spec.path}: HTTP ${measured.status}, expected ${spec.status}`); + } + + if (spec.finalPath && measured.finalPath !== spec.finalPath) { + add('final-path', `${spec.path}: landed on ${measured.finalPath}, expected ${spec.finalPath}`); + } + + if (!/^text\/html\b/i.test(measured.contentType ?? '')) { + add('not-html', `${spec.path}: content-type ${measured.contentType ?? '(none)'}`); + } + + const errorSurface = `${measured.title ?? ''} ${measured.h1 ?? ''}`.toLowerCase(); + const marker = ERROR_MARKERS.find((m) => errorSurface.includes(m)); + if (marker) add('error-shell', `${spec.path}: title/heading carries "${marker}"`); + + if (measured.text < spec.minText) { + add( + 'too-little-text', + `${spec.path}: ${measured.text} visible characters in ${measured.bytes} B, floor ${spec.minText}`, + ); + } + + if (!measured.title) add('no-title', `${spec.path}: no non-empty title element`); + + if (!measured.h1) { + add('no-h1', `${spec.path}: no non-empty h1 element`); + } else if (spec.h1 && !spec.h1.test(measured.h1)) { + add('h1-mismatch', `${spec.path}: h1 is "${measured.h1}", expected ${spec.h1}`); + } + + if (measured.links < spec.minLinks) { + add('few-links', `${spec.path}: ${measured.links} same-site links, floor ${spec.minLinks}`); + } + + if (spec.lang && measured.lang !== spec.lang) { + add('lang-mismatch', `${spec.path}: html lang="${measured.lang}", expected "${spec.lang}"`); + } + + return { findings, measured }; +} + +/* --------------------------------------------------------------- fetching -- */ + +/** + * Fetch one target, following redirects. + * + * Retries only a transport error or a 5xx, and only to absorb a flake: a page + * that is genuinely broken is still broken on the last attempt, and the last + * attempt is what gets judged. `Accept-Language: en` is sent because the site + * negotiates: without it the answer depends on what the runner's client + * happens to send, and a smoke check whose expectations move with the caller + * is not a check. + */ +async function fetchTarget(base, path, { attempts, timeoutMs, fetchImpl }) { + const url = new URL(path, base).toString(); + let last = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(url, { + redirect: 'follow', + signal: controller.signal, + headers: { + 'Accept-Language': 'en', + 'User-Agent': 'objectos-smoke-docs (+https://github.com/objectstack-ai/objectos)', + }, + }); + const body = await response.text(); + last = { + status: response.status, + url: response.url || url, + contentType: response.headers.get('content-type'), + body, + attempt, + }; + if (response.status < 500) return last; + } catch (error) { + last = { error: `${error?.name ?? 'Error'}: ${error?.message ?? error}`, attempt }; + } finally { + clearTimeout(timer); + } + if (attempt < attempts) await new Promise((r) => setTimeout(r, 2000)); + } + return last; +} + +/* ------------------------------------------------------------------- run -- */ + +function report(label, target, result) { + const m = result.measured; + const head = `${label} ${target.path}`; + if (m.error) { + console.log(`${head}\n transport: ${m.error}`); + return; + } + console.log( + `${head}\n` + + ` http ${m.status} final ${m.finalPath} ${m.contentType ?? '(no content-type)'}\n` + + ` lang=${m.lang} h1=${JSON.stringify(m.h1)} title=${JSON.stringify(m.title)}\n` + + ` ${m.bytes} B ${m.text} visible chars ${m.links} same-site links`, + ); +} + +async function run(options) { + const { + base, + targets, + negativeControlPath, + attempts = 3, + timeoutMs = 20000, + fetchImpl = fetch, + } = options; + + console.log(`smoke: ${base}`); + console.log(''); + + let failures = 0; + + for (const target of targets) { + const res = await fetchTarget(base, target.path, { attempts, timeoutMs, fetchImpl }); + const result = evaluate(target, res); + report(result.findings.length ? '✗' : '✓', target, result); + for (const f of result.findings) { + console.error(` [${f.rule}] ${f.detail}`); + failures += 1; + } + console.log(''); + } + + if (negativeControlPath) { + const control = { path: negativeControlPath, finalPath: negativeControlPath, h1: /\S/ }; + const res = await fetchTarget(base, negativeControlPath, { attempts: 1, timeoutMs, fetchImpl }); + const result = evaluate(control, res); + report(result.findings.length ? '✓ (control, expected red)' : '✗ (control)', control, result); + if (result.findings.length === 0) { + console.error( + ` [negative-control-passed] ${negativeControlPath} produced no findings — ` + + 'this check cannot currently tell a rendered page from an unrendered one, ' + + 'so its greens above mean nothing', + ); + failures += 1; + } else { + console.log( + ` control tripped [${[...new Set(result.findings.map((f) => f.rule))].join(' ')}] — ` + + 'the live path is demonstrably able to fail', + ); + } + console.log(''); + } + + if (failures) { + console.error(`✗ smoke: ${failures} finding(s) against ${base}`); + return 1; + } + console.log( + `✓ smoke: ${targets.length} page(s) rendered against ${base}` + + (negativeControlPath ? ', negative control demonstrated red' : ''), + ); + return 0; +} + +/* ------------------------------------------------------------- self-test -- */ + +/** A document with enough prose and enough sidebar to satisfy every floor. */ +function goodPage({ + lang = 'en', + title = 'Quickstart | ObjectOS', + h1 = 'Quickstart', + links = 20, + prose = 'ObjectOS is a self-hosted runtime for building internal tools. '.repeat(20), +} = {}) { + const nav = Array.from({ length: links }, (_, i) => `Page ${i}`).join(''); + return ( + `${title}` + + `` + + `

${h1}

${prose}

` + ); +} + +/** The empty shell this whole script exists for: 200, big, and not a page. */ +function emptyShell() { + return ( + 'ObjectOS' + + `
` + ); +} + +const OK_RES = (body, over = {}) => ({ + status: 200, + url: 'https://docs.objectos.ai/docs/quickstart', + contentType: 'text/html; charset=utf-8', + body, + ...over, +}); + +const BASE_TARGET = { + path: '/docs/quickstart', + finalPath: '/docs/quickstart', + h1: /^Quickstart$/i, +}; + +const CASES = [ + { name: 'a rendered page trips nothing', res: OK_RES(goodPage()), expect: [] }, + { + name: 'transport error', + res: { error: 'TypeError: fetch failed' }, + expect: ['fetch-failed'], + }, + { + name: 'not 200', + res: OK_RES(goodPage(), { status: 503 }), + expect: ['status'], + }, + { + name: 'served as plain text', + res: OK_RES(goodPage(), { contentType: 'text/plain; charset=utf-8' }), + expect: ['not-html'], + }, + { + name: 'the Next.js 404 surface', + res: OK_RES( + goodPage({ title: '404: This page could not be found.', h1: '404', links: 0, prose: '' }), + { status: 404 }, + ), + expect: ['status', 'error-shell', 'too-little-text', 'h1-mismatch', 'few-links'], + }, + { + name: 'a 200 empty shell', + res: OK_RES(emptyShell()), + expect: ['too-little-text', 'no-h1', 'few-links'], + }, + { + name: 'no title element', + res: OK_RES(goodPage().replace(/[\s\S]*?<\/title>/, '')), + expect: ['no-title'], + }, + { + name: 'the wrong page under the right URL', + res: OK_RES(goodPage({ h1: 'Architecture' })), + expect: ['h1-mismatch'], + }, + { + name: 'body without the sidebar', + res: OK_RES(goodPage({ links: 3 })), + expect: ['few-links'], + }, + { + name: 'wrong document language', + res: OK_RES(goodPage({ lang: 'zh-Hans' })), + expect: ['lang-mismatch'], + }, + { + name: 'redirected somewhere else', + res: OK_RES(goodPage(), { url: 'https://docs.objectos.ai/zh-Hans/docs/quickstart' }), + expect: ['final-path'], + }, +]; + +/** Whole-run cases, driven through `run()` with an injected fetch. */ +async function runCases() { + const results = []; + + // A negative control that renders is the failure `negative-control-passed` + // names: every target came back clean AND so did the page that must not. + const alwaysGood = async (url) => ({ + status: 200, + url, + headers: new Map([['content-type', 'text/html; charset=utf-8']]), + text: async () => goodPage({ h1: url.includes('control') ? 'Anything' : 'Quickstart' }), + }); + const asResponse = (impl) => async (url, init) => { + const r = await impl(url, init); + return { ...r, headers: { get: (k) => r.headers.get(k) } }; + }; + const quiet = () => {}; + const logs = { log: console.log, error: console.error }; + console.log = quiet; + console.error = quiet; + const code = await run({ + base: 'https://example.invalid', + targets: [BASE_TARGET], + negativeControlPath: '/docs/control', + attempts: 1, + fetchImpl: asResponse(alwaysGood), + }); + const codeTransport = await run({ + base: 'https://example.invalid', + targets: [BASE_TARGET], + negativeControlPath: null, + attempts: 1, + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + console.log = logs.log; + console.error = logs.error; + + results.push({ + name: 'a negative control that renders fails the run', + ok: code === 1, + rule: 'negative-control-passed', + }); + results.push({ + name: 'a transport error fails the run', + ok: codeTransport === 1, + rule: 'fetch-failed', + }); + return results; +} + +async function selfTest() { + let failed = 0; + + for (const c of CASES) { + const { findings } = evaluate(BASE_TARGET, c.res); + const fired = [...new Set(findings.map((f) => f.rule))].sort(); + const want = [...c.expect].sort(); + const ok = fired.join(',') === want.join(','); + if (!ok) failed += 1; + console.log( + `${ok ? '✓' : '✗'} ${c.name.padEnd(38)} fired [${fired.join(' ') || '—'}]` + + (ok ? '' : ` expected [${want.join(' ') || '—'}]`), + ); + if (!ok) for (const f of findings) console.error(` [${f.rule}] ${f.detail}`); + } + + console.log(''); + const runResults = await runCases(); + for (const r of runResults) { + if (!r.ok) failed += 1; + console.log(`${r.ok ? '✓' : '✗'} ${r.name}`); + } + + console.log(''); + const covered = new Set([...CASES.flatMap((c) => c.expect), ...runResults.map((r) => r.rule)]); + for (const rule of RULES) { + if (!covered.has(rule)) { + console.error(`✗ rule "${rule}" has no fixture that trips it`); + failed += 1; + } + } + // A rule that fires on the clean fixture would make every run red for the + // wrong reason, so silence on a rendered page is asserted, not assumed. + const cleanCase = CASES.find((c) => c.expect.length === 0); + if (!cleanCase) { + console.error('✗ no fixture asserts that a rendered page trips nothing'); + failed += 1; + } + + if (failed) { + console.error(`\n✗ self-test: ${failed} case(s) did not behave as declared`); + process.exitCode = 1; + return; + } + console.log( + `✓ self-test: ${CASES.length} response case(s) and ${runResults.length} run case(s) — ` + + `all ${RULES.length} rules demonstrated able to fail`, + ); +} + +/* ------------------------------------------------------------------ main -- */ + +function parseArgs(argv) { + const opts = { base: process.env.SMOKE_BASE_URL || DEFAULT_BASE, paths: null, control: NEGATIVE_CONTROL_PATH }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + if (a === '--base') opts.base = argv[++i]; + else if (a === '--paths') opts.paths = argv[++i]; + else if (a === '--negative-control') opts.control = argv[++i]; + else if (a === '--no-negative-control') opts.control = null; + else if (a === '--self-test') opts.selfTest = true; + else { + console.error(`unknown argument: ${a}`); + process.exit(2); + } + } + return opts; +} + +async function main() { + const argv = process.argv.slice(2); + if (argv.some((a) => a === '--self-test')) return selfTest(); + + const opts = parseArgs(argv); + const targets = opts.paths + ? opts.paths + .split(',') + .map((p) => p.trim()) + .filter(Boolean) + .map((p) => ({ path: p, finalPath: p, h1: /\S/ })) + : TARGETS; + + if (targets.length === 0) { + console.error('✗ smoke: no targets — refusing to report a green on an empty check'); + process.exitCode = 1; + return; + } + + process.exitCode = await run({ + base: opts.base.replace(/\/+$/, ''), + targets, + negativeControlPath: opts.control, + }); +} + +await main(); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fda9fd0..3857d0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,26 @@ jobs: build: runs-on: ubuntu-latest + # `NEXT_PRIVATE_STANDALONE` is what `@opennextjs/aws` sets before it runs + # `next build` — its own comment reads "Equivalent to setting `output: + # "standalone"` in next.config.js". Without it a plain `next build` + # produces no `.next/standalone/`, and the packaging step below fails on a + # missing `pages-manifest.json` three directories inside it. Measured on + # this branch before it was set: `ENOENT ... .next/standalone/apps/docs/ + # .next/server/pages-manifest.json`. + # + # Set for the whole job rather than for the deploy path only, so that what + # a pull request builds is the same shape as what gets published. A build + # that differs from the deploy build is a small instance of the defect this + # card is about. + # + # It has to be declared in `turbo.json` as well: turbo 2 runs tasks in + # strict env mode, so an undeclared variable never reaches `next build` — + # and declaring it is also what puts it in the cache key, so a `.next` + # cached from before this line cannot be replayed without the standalone + # tree the packaging step needs. + env: + NEXT_PRIVATE_STANDALONE: 'true' steps: - uses: actions/checkout@v7 - uses: pnpm/action-setup@v6 @@ -116,3 +136,70 @@ jobs: run: node .github/scripts/check-locale-surface.mjs | tee -a "$GITHUB_STEP_SUMMARY" - run: pnpm turbo run test + + # Defect 2 of #269: the deploy used to run its own `pnpm install` and its + # own `opennextjs-cloudflare build`, so CI built the site, threw it away, + # and the deploy published a SECOND build that nothing here had checked. + # The published artifact was unverified by construction. + # + # `--skipNextBuild` packages the `.next` output `pnpm turbo run build` + # produced above — the same output `Locale surface` measured and the same + # tree every step in this job passed — and `deploy-docs.yml` uploads THIS + # bundle rather than making another one. + # + # Last in the job on purpose: the artifact then only exists for a commit + # that cleared every gate above it. And only on a push to `main`, because + # that is the only event that deploys, so a pull request pays nothing. + - name: Package the Worker from the build this job tested + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + working-directory: apps/docs + run: pnpm exec opennextjs-cloudflare build --skipNextBuild + + # `include-hidden-files` is load-bearing, not tidiness: the compiled + # OpenNext config the deploy reads lives at `.open-next/.build/`, and + # upload-artifact excludes dotted paths by default. Without it the + # download succeeds, the deploy exits 1 on a missing config, and the + # cause is three directories away from the message. + - name: Upload the Worker bundle + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v7 + with: + name: docs-worker + path: apps/docs/.open-next + include-hidden-files: true + if-no-files-found: error + retention-days: 3 + + # Defect 1 of #269: `deploy-docs.yml` used to hang off `push: branches: + # [main]` exactly as this workflow does, so the two ran in PARALLEL and a + # commit that failed any gate above still deployed. There was no `needs:` and + # no `workflow_run` anywhere. + # + # As a job here it cannot start until `node-floor` and `build` are green, and + # the `if:` keeps it off pull requests and merge groups. `workflow_run` would + # also have gated it, but it fires on a FAILED run too — the conclusion has + # to be re-checked by hand inside the workflow — and it runs detached from + # the run whose artifact it publishes, which is what the `with:` line here + # depends on. + # + # ⚠️ This deploy is expected to FAIL while #261 is open: `main` builds a + # Worker over Cloudflare's 64 MiB limit, so the upload is rejected at version + # creation and the serving version cannot be displaced. That is the current + # deliberate steady state, not a regression from this wiring. + deploy-docs: + name: Deploy docs + needs: [node-floor, build] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # Preserves what `deploy-docs.yml` declared for itself before it became a + # called workflow: one deploy at a time, and never cancel one in flight. + concurrency: + group: deploy-docs + cancel-in-progress: false + permissions: + contents: read + actions: write # dispatch rollback-docs.yml when the smoke check fails + issues: write # file or update the one deploy-failure card + uses: ./.github/workflows/deploy-docs.yml + with: + artifact_name: docs-worker + secrets: inherit diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 5d34bb7..d40b182 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -1,27 +1,102 @@ name: Deploy Docs +# The deploy, and the verification layer around it (#269). +# +# ## Why this file no longer has a `push:` trigger +# +# It used to hang off `push: branches: [main]` exactly as `ci.yml` does, which +# meant the two ran in PARALLEL: a commit that failed `build`, `Node floor`, +# the locale surface or any translation gate still deployed. There was no +# `needs:`, no `workflow_run`, nothing. That is defect 1 of #269. +# +# The fix is that this workflow is now CALLED by `ci.yml` as a job with +# `needs: [node-floor, build]`, so it cannot start until those are green, and +# `ci.yml` restricts it to a push on `main`. `workflow_run` was the alternative +# and was rejected: it fires on a failed run too, so the conclusion has to be +# re-checked by hand inside the workflow, and it runs detached from the run +# whose artifact it is meant to publish — which would have made defect 2 below +# unfixable without re-downloading across runs. +# +# `workflow_dispatch` survives, but it can no longer deploy. It runs the smoke +# check against whatever is live, which is the one thing a human wants on +# demand; publishing outside the CI gate is exactly the hole this card closes. +# +# ## What the jobs are for +# +# deploy publishes the bundle CI built and tested (defect 2), and refuses +# to call it a success without a NEW version id serving (defect 4) +# smoke asks the live site whether it renders (defect 3) +# rollback reuses rollback-docs.yml (#267) when the smoke check goes red +# report files or updates ONE issue on any failure, because this lane reads +# the board and not the run list — 36 red runs over 10 days produced +# no card at all +# +# ⚠️ Every deploy is currently REJECTED: `main` builds a Worker over +# Cloudflare's 64 MiB limit (#261), so `deploy` is expected to go red and the +# serving version cannot be displaced. `smoke` still runs — it is checking the +# live site, not this run's output — which is why it is not gated on the deploy +# succeeding. + on: - push: - branches: [main] - paths: - - 'apps/docs/**' - - 'content/docs/**' - - 'pnpm-lock.yaml' - - '.github/workflows/deploy-docs.yml' + workflow_call: + inputs: + artifact_name: + description: 'Name of the artifact holding the .open-next bundle CI built and tested.' + required: true + type: string + base_url: + description: 'Origin the smoke check runs against.' + required: false + type: string + default: 'https://docs.objectos.ai' + file_issue: + description: 'File or update the deploy-failure card when something goes red.' + required: false + type: boolean + default: true workflow_dispatch: - -concurrency: - group: deploy-docs - cancel-in-progress: false + inputs: + # Declared so that `inputs.artifact_name` is a defined property under both + # triggers rather than one. It is deliberately unusable from here: a + # manual run has no artifact from a CI run to publish, and the `deploy` + # job below refuses to start on a `workflow_dispatch` event regardless of + # what this says. Publishing outside the CI gate is the hole #269 closes. + artifact_name: + description: 'Leave blank. A manual run cannot deploy; it only smoke-checks whatever is live.' + required: false + type: string + default: '' + base_url: + description: 'Origin to smoke-check. Defaults to the live docs site.' + required: false + type: string + default: 'https://docs.objectos.ai' + paths: + description: 'Comma-separated paths to check INSTEAD of the built-in target list. Use this to point the check at something that does not render and watch it go red.' + required: false + type: string + default: '' + file_issue: + description: 'File or update the deploy-failure card if this run goes red. Off by default: a manual run is usually an experiment.' + required: false + type: boolean + default: false jobs: deploy: + name: Publish the tested Worker + # Only when a caller handed us a bundle. A manual dispatch has no + # `artifact_name` and therefore cannot deploy — see the header. + if: github.event_name != 'workflow_dispatch' && inputs.artifact_name != '' runs-on: ubuntu-latest environment: name: cloudflare-docs url: https://docs-objectos.objectstack.workers.dev permissions: contents: read + outputs: + previous_version_id: ${{ steps.verdict.outputs.previous_version_id }} + new_version_id: ${{ steps.verdict.outputs.new_version_id }} steps: - uses: actions/checkout@v7 - uses: pnpm/action-setup@v6 @@ -30,9 +105,230 @@ jobs: node-version: 22 cache: pnpm - run: pnpm install --frozen-lockfile - - name: Deploy to Cloudflare Workers + + # Defect 2 of #269: the deploy used to run `opennextjs-cloudflare build` + # itself, so CI built the site, threw it away, and the deploy published a + # DIFFERENT build that nothing had checked. This downloads the bundle the + # `build` job produced from the `.next` output its own gates measured. + # `opennextjs-cloudflare deploy` does not build; it uploads what is here. + - name: Download the Worker CI built and tested + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.artifact_name }} + path: apps/docs/.open-next + + # Taken BEFORE the deploy and required to succeed: it is both half of the + # "did anything actually change" comparison and the rollback target. A + # deploy with no known-good version to fall back to is not one this + # pipeline should start. + - name: Read the version that is serving now + working-directory: apps/docs + shell: bash + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + run: | + pnpm exec wrangler deployments status --name docs-objectos --json \ + > "$RUNNER_TEMP/before.json" + cat "$RUNNER_TEMP/before.json" + + # `set +e` is the point of this step, not an oversight. #269: "Do not + # treat a step's exit code as evidence the deploy worked." A rejected + # upload and an accepted one are not reliably distinguishable from out + # here, so the exit code is captured as EVIDENCE and handed to the gate + # below, which is the only thing allowed to call this a success. + - name: Deploy to Cloudflare + id: deploy + working-directory: apps/docs + shell: bash + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + run: | + set +e + pnpm exec opennextjs-cloudflare deploy > "$RUNNER_TEMP/deploy.log" 2>&1 + DEPLOY_EXIT=$? + set -e + cat "$RUNNER_TEMP/deploy.log" + echo "exit_code=$DEPLOY_EXIT" >> "$GITHUB_OUTPUT" + echo "::notice::deploy command exited ${DEPLOY_EXIT} — the verdict is the next step, not this number" + + - name: Read what is serving afterwards + if: always() working-directory: apps/docs + shell: bash env: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - run: pnpm run deploy + run: | + set +e + pnpm exec wrangler deployments status --name docs-objectos --json \ + > "$RUNNER_TEMP/after.json" 2>"$RUNNER_TEMP/after.err" + set -e + cat "$RUNNER_TEMP/after.json" "$RUNNER_TEMP/after.err" + + # The arbiter. Requires a new, well-formed version id that was not + # already serving and that the post-deploy reading agrees is serving now. + # An unreadable input is a finding here, never a skip. + # + # `shell: bash` is load-bearing, not tidiness: the default shell for a + # `run:` step is `bash -e {0}` with no pipefail, so in `node ... | tee` + # the step takes tee's status and a gate that exits 1 passes silently. + - name: Assert a new version is serving + id: verdict + if: always() + shell: bash + run: | + node .github/scripts/check-deploy-version.mjs \ + --before "$RUNNER_TEMP/before.json" \ + --after "$RUNNER_TEMP/after.json" \ + --deploy-log "$RUNNER_TEMP/deploy.log" \ + --deploy-exit "${{ steps.deploy.outputs.exit_code || '1' }}" \ + | tee -a "$GITHUB_STEP_SUMMARY" + + smoke: + name: Smoke-check the live site + needs: [deploy] + # Runs whether or not the deploy published anything, and that is deliberate + # while #261 is open: today every deploy is rejected, so gating this on a + # successful deploy would mean the site is never checked at all. The + # rollback below is what is gated on the deploy having succeeded. + if: always() && needs.deploy.result != 'cancelled' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 22 + + # Zero-dependency on purpose: no install, so this stays a seconds-long + # job and can be run by hand against any origin. + # + # Every run also fetches a path that must NOT render and requires it to + # produce findings — see the script's header. A green here therefore + # carries a live demonstration that the same code path can go red, which + # is the only thing that makes it worth reading. + - name: Smoke + shell: bash + env: + BASE_URL: ${{ inputs.base_url }} + SMOKE_PATHS: ${{ inputs.paths }} + run: | + ARGS=(--base "$BASE_URL") + if [ -n "$SMOKE_PATHS" ]; then + ARGS+=(--paths "$SMOKE_PATHS") + echo "::warning::running against an ad-hoc path list: $SMOKE_PATHS" + fi + node .github/scripts/smoke-docs.mjs "${ARGS[@]}" | tee -a "$GITHUB_STEP_SUMMARY" + + rollback: + name: Roll back the bad version + needs: [deploy, smoke] + # Only when THIS run published something and the site then failed to + # render. A failed deploy displaces nothing, so there is nothing to undo. + if: >- + always() + && needs.deploy.result == 'success' + && needs.smoke.result == 'failure' + && needs.deploy.outputs.previous_version_id != '' + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@v7 + # Reusing rollback-docs.yml (#267) rather than reimplementing the + # Cloudflare call: it already validates the version id, already draws the + # same `cloudflare-docs` credentials, and a required-reviewer rule added + # to that environment (#266) is meant to gate this path too. + - name: Dispatch rollback-docs.yml + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + VERSION_ID: ${{ needs.deploy.outputs.previous_version_id }} + BAD_VERSION: ${{ needs.deploy.outputs.new_version_id }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh workflow run rollback-docs.yml --ref "$DEFAULT_BRANCH" \ + -f version_id="$VERSION_ID" \ + -f reason="automatic rollback: smoke check failed after $BAD_VERSION ($RUN_URL)" + echo "::notice::rollback to $VERSION_ID dispatched" + + report: + name: File or update the failure card + needs: [deploy, smoke, rollback] + if: >- + always() + && inputs.file_issue + && (needs.deploy.result == 'failure' + || needs.smoke.result == 'failure' + || needs.rollback.result == 'failure') + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + # Defect 4 of #269: 36 consecutive red deploys across two PM tenures + # produced no card. This lane reads the board every round and does not + # read the run list, so a red run is not a signal — a card is. + # + # ONE card, updated. `gh issue list --label` finds the open one; a new + # card is only created when a human has closed the last one, which is the + # correct reading of "this is fixed, tell me if it comes back". + - name: File or update + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + DEPLOY_RESULT: ${{ needs.deploy.result }} + SMOKE_RESULT: ${{ needs.smoke.result }} + ROLLBACK_RESULT: ${{ needs.rollback.result }} + NEW_VERSION: ${{ needs.deploy.outputs.new_version_id }} + PREV_VERSION: ${{ needs.deploy.outputs.previous_version_id }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SHA: ${{ github.sha }} + shell: bash + run: | + set -euo pipefail + + LABEL='deploy-failure' + TITLE='The docs deploy pipeline is failing' + + gh label create "$LABEL" \ + --color 'B60205' \ + --description 'Automated: the docs deploy or its post-deploy smoke check failed' \ + --force + + NUM="$(gh issue list --label "$LABEL" --state open --limit 1 --json number --jq '.[0].number // empty')" + + BODY="$(cat <<EOF + Automated report from \`ci.yml\` -> \`deploy-docs.yml\`. + + | | | + |:--|:--| + | run | $RUN_URL | + | commit | \`$SHA\` | + | deploy | \`$DEPLOY_RESULT\` | + | smoke | \`$SMOKE_RESULT\` | + | rollback | \`$ROLLBACK_RESULT\` | + | version serving before | \`${PREV_VERSION:-unknown}\` | + | version published | \`${NEW_VERSION:-none}\` | + + A \`deploy\` failure with no published version usually means Cloudflare + refused the upload — while #261 is open that is the expected steady + state and the live site is unaffected. A \`smoke\` failure means the + live site did not render; the run log names which rule fired. + EOF + )" + + if [ -n "$NUM" ]; then + gh issue comment "$NUM" --body "$BODY" + echo "::notice::updated existing card #$NUM" + else + URL="$(gh issue create --title "$TITLE" --label "$LABEL" --body "$BODY")" + echo "::notice::filed $URL" + fi + diff --git a/tools/ci-scripts/run-self-tests.mjs b/tools/ci-scripts/run-self-tests.mjs index f5dd011..ee0d4cc 100644 --- a/tools/ci-scripts/run-self-tests.mjs +++ b/tools/ci-scripts/run-self-tests.mjs @@ -61,6 +61,14 @@ * cached green is the exact failure it exists to end. So `ci.yml` invokes the * gate itself as a step after `pnpm turbo run build`, and what runs here is * the fixture-driven proof that its rules can still go red. + * + * `check-deploy-version.mjs` and `smoke-docs.mjs` are here on the same footing + * and for a sharper reason (#269). Their gate modes need a Cloudflare + * credential and a live website respectively, so on a pull request neither one + * can run for real — and the deploy path they guard is currently REJECTED at + * Cloudflare, so on `main` they may not run for real either. What keeps them + * from being decoration in the meantime is exactly this: their fixtures assert + * that every rule they enforce still produces a red, and that runs on every PR. */ import { readFileSync, readdirSync, existsSync } from 'node:fs'; @@ -78,6 +86,8 @@ const SELF_TESTED = [ 'check-translations.mjs', 'check-node-floor.mjs', 'check-locale-surface.mjs', + 'check-deploy-version.mjs', + 'smoke-docs.mjs', ]; /** diff --git a/turbo.json b/turbo.json index 181fd11..ab9ead0 100644 --- a/turbo.json +++ b/turbo.json @@ -4,6 +4,7 @@ "tasks": { "build": { "dependsOn": ["^build"], + "env": ["NEXT_PRIVATE_STANDALONE"], "inputs": ["$TURBO_DEFAULT$", "$TURBO_ROOT$/content/docs/**"], "outputs": ["dist/**", ".next/**", "!.next/cache/**"] },