diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 527d7de..4c3cc52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,23 @@ jobs: esac node -e "const r=JSON.parse(process.env.REPORT);if(r.overall.total!==Number(process.env.SCORE))process.exit(1)" + - name: Run the copyable end-to-end sample + id: sample + uses: ./ + with: + path: examples/github-action-sample/site + package-spec: ./aeoptimize-0.6.0.tgz + + - name: Validate sample outputs + env: + SCORE: ${{ steps.sample.outputs.score }} + REPORT: ${{ steps.sample.outputs.report }} + run: | + case "$SCORE" in + ''|*[!0-9]*) echo "Invalid sample score: $SCORE"; exit 1 ;; + esac + node -e "const r=JSON.parse(process.env.REPORT);if(r.pages.length!==1||r.overall.total!==Number(process.env.SCORE))process.exit(1)" + - name: Blocking mode passes at an accepted threshold id: blocking-passes uses: ./ diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e65f1..ac51e8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable user-visible changes will be documented here. The project follows Semantic Versioning after the v0.6 evidence baseline is released. -## 0.6.0 — 2026-08-16 +## 0.6.0 ### Changed @@ -14,6 +14,8 @@ All notable user-visible changes will be documented here. The project follows Se - Limited CI support to maintained Node.js LTS lines and refreshed dependencies. - Added methodology, contribution, security, roadmap, and root GitHub Action files. - Made the GitHub Action advisory by default, with explicit blocking mode, version-matched package installation, stable outputs, and contract fixtures. +- Added a public positive, negative, and false-positive boundary corpus for every scored rule. +- Added a copyable end-to-end GitHub Action sample plus release and rollback instructions. ### Security diff --git a/README.md b/README.md index c750349..0d49afa 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Two often-promoted AEO signals are deliberately excluded from the score: - FAQ content and `FAQPage` schema are optional. The generator does not infer FAQ schema from question headings. - `llms.txt` is an experimental proposal. Generating or publishing it does not add points. -Every rule, its evidence class, and known false-positive boundary is documented in [docs/methodology.md](docs/methodology.md). +Every rule, its evidence class, and known false-positive boundary is documented in [docs/methodology.md](docs/methodology.md) and exercised by the [versioned public fixture corpus](fixtures/v0.6/rule-corpus.ts). ## CI contract @@ -81,6 +81,8 @@ Projects can explicitly choose blocking mode after accepting a baseline: The Action exposes `score` and `report` outputs in both modes. Its release is reproducible only when the Action tag and matching npm package version both exist. Before pinning a version, verify both artifacts; if either is missing, use the CLI directly. +A copyable advisory workflow and controlled input are available in the [end-to-end Action sample](examples/github-action-sample/README.md). + ## Optional generators ```bash @@ -149,7 +151,7 @@ claude plugin marketplace add cucuwang/aeoptimize ## Project status -The next evidence release focuses on methodology, reproducible fixtures, CI compatibility, packaging, and external adoption—not more scoring rules. See [ROADMAP.md](ROADMAP.md). +The v0.6 evidence baseline focuses on methodology, reproducible fixtures, CI compatibility, packaging, and external adoption—not more scoring rules. Release acceptance and rollback are documented in [docs/release-v0.6.md](docs/release-v0.6.md); longer-term adoption work remains in [ROADMAP.md](ROADMAP.md). Contributions are welcome. Rule changes require an evidence note and positive/negative fixtures; see [CONTRIBUTING.md](CONTRIBUTING.md). Report vulnerabilities through the process in [SECURITY.md](SECURITY.md). diff --git a/ROADMAP.md b/ROADMAP.md index aceaecd..3589a56 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,6 +14,8 @@ Release gates: - root GitHub Action metadata and an end-to-end sample repository; - stable JSON output, changelog, release notes, and rollback instructions. +The release gates are enforced by `src/core/__tests__/release-contract.test.ts`, the versioned corpus in `fixtures/v0.6/`, the copyable sample in `examples/github-action-sample/`, and the release runbook in `docs/release-v0.6.md`. + ## Product validation after v0.6 - Validate one core workflow: static site or documentation CI content-readiness lint. diff --git a/action/test-contract.sh b/action/test-contract.sh index 030257a..825768c 100644 --- a/action/test-contract.sh +++ b/action/test-contract.sh @@ -4,6 +4,7 @@ set -euo pipefail REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) RUNNER="$REPO_ROOT/action/run.sh" FIXTURE="$REPO_ROOT/.github/fixtures/action-low" +SAMPLE_FIXTURE="$REPO_ROOT/examples/github-action-sample/site" CLI="$REPO_ROOT/dist/cli/index.js" TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/aeoptimize-action-contract.XXXXXX") trap 'rm -rf "$TEST_ROOT"' EXIT @@ -13,10 +14,11 @@ run_case() { local fail_on_low_score=$2 local min_score=$3 local expected_status=$4 + local input_path=${5:-$FIXTURE} local output_file="$TEST_ROOT/$name.output" local status=0 - INPUT_PATH="$FIXTURE" \ + INPUT_PATH="$input_path" \ MIN_SCORE="$min_score" \ FAIL_ON_LOW_SCORE="$fail_on_low_score" \ GITHUB_OUTPUT="$output_file" \ @@ -36,11 +38,14 @@ run_case blocking-passes true 0 0 run_case invalid-threshold false invalid 2 run_case invalid-threshold-high false 101 2 run_case invalid-choice sometimes 60 2 +run_case sample-advisory false 100 0 "$SAMPLE_FIXTURE" grep -Eq '^score=[0-9]+$' "$TEST_ROOT/advisory.output" grep -q '^report< +``` + +## Rollback + +An npm dist-tag rollback changes what `npm install aeoptimize` selects; it does not remove exact-version installs. Never silently move an existing Git tag to different code. + +If npm 0.6.0 is unsuitable before a corrective release is available, request separate authorization for each mutation, then: + +```bash +npm dist-tag add aeoptimize@0.5.3 latest +npm deprecate aeoptimize@0.6.0 "Use 0.5.3 until the corrective release is available." +``` + +Mark the GitHub Release with the same warning. Preserve the `v0.6.0` tag as evidence of what was published, fix forward in 0.6.1, rerun the complete release acceptance suite, and only then move npm `latest` to the corrective version. diff --git a/examples/github-action-sample/.github/workflows/aeoptimize.yml b/examples/github-action-sample/.github/workflows/aeoptimize.yml new file mode 100644 index 0000000..75eecb1 --- /dev/null +++ b/examples/github-action-sample/.github/workflows/aeoptimize.yml @@ -0,0 +1,18 @@ +name: Content readiness + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + readiness: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: cucuwang/aeoptimize@v0.6.0 + with: + path: site + fail-on-low-score: 'false' diff --git a/examples/github-action-sample/README.md b/examples/github-action-sample/README.md new file mode 100644 index 0000000..356246f --- /dev/null +++ b/examples/github-action-sample/README.md @@ -0,0 +1,9 @@ +# aeoptimize GitHub Action sample + +This directory is a copyable end-to-end sample for the v0.6 Action contract. + +- `.github/workflows/aeoptimize.yml` checks the static site on pull requests and manual runs. +- `site/index.html` is a deterministic public input. +- The Action is advisory by default. The sample does not block a pull request on an unreviewed score threshold. + +The workflow becomes reproducible only after both `aeoptimize@0.6.0` exists on npm and the immutable `v0.6.0` Git tag points to the matching release commit. Until both artifacts exist, use the local CLI or the release-candidate package during controlled verification. diff --git a/examples/github-action-sample/site/index.html b/examples/github-action-sample/site/index.html new file mode 100644 index 0000000..6c16c45 --- /dev/null +++ b/examples/github-action-sample/site/index.html @@ -0,0 +1,38 @@ + + + + + + + + + aeoptimize Action sample + + + +
+
+

aeoptimize Action sample

+

The aeoptimize Action sample is a controlled input for the public v0.6 automation contract.

+

What the workflow verifies

+

The workflow installs the version-matched package, scans this directory, and exposes a numeric score plus a JSON report.

+
    +
  • Advisory mode reports findings without blocking a pull request.
    • +
  • Blocking mode requires a project owner to accept a baseline first.
    • +
  • The score remains a content-readiness heuristic rather than an outcome prediction.
    • +
+

How to interpret results

+

According to the versioned methodology, teams should compare reports only within the same project, package version, configuration, and fixture set.

+ 
npx aeoptimize scan ./site --dir --json
+
+
+ + diff --git a/fixtures/v0.6/rule-corpus.ts b/fixtures/v0.6/rule-corpus.ts new file mode 100644 index 0000000..ebd9c2d --- /dev/null +++ b/fixtures/v0.6/rule-corpus.ts @@ -0,0 +1,314 @@ +export type RuleFixtureKind = 'positive' | 'negative' | 'boundary'; + +export interface FixtureDocument { + url: string; + title: string; + html: string; + markdown: string; + headings: Array<{ level: number; text: string }>; + paragraphs: string[]; + jsonLd: Array<{ '@type'?: string; '@context'?: string; [key: string]: unknown }>; + metaTags: Record; + links: Array<{ href: string; text: string; rel?: string }>; + rawText: string; +} + +export interface RuleFixtureCase { + purpose: string; + document: Partial; + expected: { + score: number; + issues: number; + suggestions: number; + }; +} + +export interface RuleFixtureSet { + positive: RuleFixtureCase; + negative: RuleFixtureCase; + boundary: RuleFixtureCase; +} + +const words = (count: number, prefix = 'word') => + Array.from({ length: count }, (_, index) => `${prefix}${index}`).join(' '); + +const repeatedWords = (count: number, word = 'word') => + Array.from({ length: count }, () => word).join(' '); + +const stuffedText = ( + 'Buy cheap widgets now. Cheap widgets are the best widgets. ' + + 'Our widgets are cheap widgets for sale. Get cheap widgets today. ' + + 'Cheap widgets online cheap widgets store cheap widgets deals. ' + + 'Best cheap widgets cheap widgets review cheap widgets comparison. ' + + 'Order cheap widgets cheap widgets shipping cheap widgets discount. ' +).repeat(3); + +export const ruleFixtureCorpusVersion = '0.6.0'; + +export const ruleFixtureCorpus: Record = { + 'heading-hierarchy': { + positive: { + purpose: 'A descriptive H1 followed by nested sections receives the full structure score.', + document: { + headings: [ + { level: 1, text: 'Release guide' }, + { level: 2, text: 'Verification' }, + { level: 3, text: 'CLI checks' }, + ], + rawText: 'A short release guide with a clear outline.', + }, + expected: { score: 10, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'A document with no headings triggers the deterministic missing-outline finding.', + document: { headings: [], rawText: 'Unstructured content.' }, + expected: { score: 0, issues: 1, suggestions: 0 }, + }, + boundary: { + purpose: 'Multiple H1 elements are not treated as an automatic error when the outline does not skip levels.', + document: { + headings: [ + { level: 1, text: 'Primary title' }, + { level: 1, text: 'Secondary region title' }, + { level: 2, text: 'Details' }, + ], + rawText: 'Readable content.', + }, + expected: { score: 10, issues: 0, suggestions: 0 }, + }, + }, + 'paragraph-length': { + positive: { + purpose: 'Short focused paragraphs remain below the configured readability heuristic.', + document: { paragraphs: ['A concise paragraph.', 'Another concise paragraph.'] }, + expected: { score: 8, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'A majority of paragraphs above 150 words triggers the configured long-paragraph finding.', + document: { paragraphs: [repeatedWords(151), repeatedWords(151), 'A short paragraph.'] }, + expected: { score: 3, issues: 1, suggestions: 1 }, + }, + boundary: { + purpose: 'Exactly 150 words is the non-penalized threshold boundary.', + document: { paragraphs: [repeatedWords(150)] }, + expected: { score: 8, issues: 0, suggestions: 0 }, + }, + }, + 'list-usage': { + positive: { + purpose: 'Long content containing a genuine list receives the full scannability score.', + document: { html: `
  • First
  • Second
${words(301)}`, rawText: words(301) }, + expected: { score: 7, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Long content with no list receives a low-impact review suggestion.', + document: { html: `

${words(301)}

`, rawText: words(301) }, + expected: { score: 4, issues: 0, suggestions: 1 }, + }, + boundary: { + purpose: 'Content at exactly 300 words is not forced into a list merely to satisfy the heuristic.', + document: { html: `

${words(300)}

`, rawText: words(300) }, + expected: { score: 7, issues: 0, suggestions: 0 }, + }, + }, + 'self-contained-statements': { + positive: { + purpose: 'Paragraphs that name their subject remain independently understandable.', + document: { + paragraphs: [ + 'Aeoptimize reports deterministic content-readiness findings.', + 'The GitHub Action is advisory by default.', + 'Project owners choose whether a threshold should block CI.', + ], + }, + expected: { score: 8, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'A majority of dangling pronoun or transition openings triggers a review finding.', + document: { + paragraphs: ['This is important.', 'They require context.', 'However, it varies.', 'The release is versioned.', 'The report is public.'], + }, + expected: { score: 3, issues: 1, suggestions: 1 }, + }, + boundary: { + purpose: 'Exactly twenty percent dangling openings is the non-penalized ratio boundary.', + document: { + paragraphs: ['This needs context.', 'The package is versioned.', 'The Action is advisory.', 'The report is stable.', 'The fixture is public.'], + }, + expected: { score: 8, issues: 0, suggestions: 1 }, + }, + }, + 'data-stats-presence': { + positive: { + purpose: 'A quantitative claim with explicit source language is not flagged as unsourced.', + document: { rawText: 'According to the linked release report, 20 users completed the test.' }, + expected: { score: 7, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'A quantitative claim without a detectable source receives an evidence warning.', + document: { rawText: 'The package serves 20 users.' }, + expected: { score: 3, issues: 1, suggestions: 1 }, + }, + boundary: { + purpose: 'Content without quantitative claims is not penalized or encouraged to invent numbers.', + document: { rawText: 'The package exposes a deterministic local lint.' }, + expected: { score: 7, issues: 0, suggestions: 0 }, + }, + }, + 'clear-definitions': { + positive: { + purpose: 'Several explicit definitions receive the full clarity score.', + document: { rawText: 'A lint is a repeatable check. A fixture means a controlled input. A release refers to a published version.' }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Content without definitions receives a clarity suggestion.', + document: { rawText: 'Install the package and run the command.' }, + expected: { score: 1, issues: 0, suggestions: 1 }, + }, + boundary: { + purpose: 'A semantic definition list is accepted without requiring a prose pattern.', + document: { html: '
Fixture
A controlled input.
', rawText: 'Fixture: a controlled input.' }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + }, + attribution: { + positive: { + purpose: 'Accurate author, date, and source language receive the full attribution score.', + document: { + metaTags: { author: 'Fixture Author', date: '2026-08-22' }, + rawText: 'According to the release evidence, the focused checks passed.', + }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Authored or time-sensitive content with no attribution signals receives a suggestion.', + document: { metaTags: {}, rawText: 'A time-sensitive release note.' }, + expected: { score: 0, issues: 0, suggestions: 1 }, + }, + boundary: { + purpose: 'Author plus source language reaches the no-suggestion threshold without inventing a date.', + document: { metaTags: { author: 'Fixture Author' }, rawText: 'Source: local release verification.' }, + expected: { score: 3, issues: 0, suggestions: 0 }, + }, + }, + 'json-ld-presence': { + positive: { + purpose: 'Present JSON-LD is detected without awarding extra points for schema count.', + document: { jsonLd: [{ '@context': 'https://schema.org', '@type': 'SoftwareApplication' }] }, + expected: { score: 8, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Missing JSON-LD produces an informational finding but no score penalty because schema is optional.', + document: { jsonLd: [] }, + expected: { score: 8, issues: 1, suggestions: 0 }, + }, + boundary: { + purpose: 'Presence and completeness are separate rules, preventing a duplicate penalty in the presence rule.', + document: { jsonLd: [{}] }, + expected: { score: 8, issues: 0, suggestions: 0 }, + }, + }, + 'json-ld-completeness': { + positive: { + purpose: 'JSON-LD with context and type receives the full completeness score.', + document: { jsonLd: [{ '@context': 'https://schema.org', '@type': 'SoftwareApplication' }] }, + expected: { score: 12, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'A JSON-LD object missing both required fields receives the deterministic completeness warning.', + document: { jsonLd: [{}] }, + expected: { score: 0, issues: 1, suggestions: 0 }, + }, + boundary: { + purpose: 'No structured data receives no completeness penalty because optional absence belongs to the presence rule.', + document: { jsonLd: [] }, + expected: { score: 12, issues: 0, suggestions: 0 }, + }, + }, + 'robots-txt-ai-config': { + positive: { + purpose: 'An indexable page receives the full page-level crawler score.', + document: { metaTags: { robots: 'index,follow' } }, + expected: { score: 8, issues: 0, suggestions: 1 }, + }, + negative: { + purpose: 'A noindex directive triggers the deterministic critical finding.', + document: { metaTags: { robots: 'noindex,nofollow' } }, + expected: { score: 0, issues: 1, suggestions: 0 }, + }, + boundary: { + purpose: 'A nofollow-only directive is not confused with noindex; site-level crawler access remains a separate check.', + document: { metaTags: { robots: 'nofollow,noarchive' } }, + expected: { score: 8, issues: 0, suggestions: 1 }, + }, + }, + 'meta-description-quality': { + positive: { + purpose: 'A page-specific readable summary receives the full metadata score.', + document: { metaTags: { description: 'A deterministic release guide covering package, Action, and rollback verification.' } }, + expected: { score: 7, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'A missing description receives a warning without claiming a ranking outcome.', + document: { metaTags: {} }, + expected: { score: 0, issues: 1, suggestions: 0 }, + }, + boundary: { + purpose: 'A long but page-specific description is not penalized by a fabricated fixed-length limit.', + document: { metaTags: { description: `A page-specific release explanation ${repeatedWords(180, 'context')}.` } }, + expected: { score: 7, issues: 0, suggestions: 0 }, + }, + }, + 'content-boilerplate-ratio': { + positive: { + purpose: 'Paragraph content at sixty percent of extracted text receives the full heuristic score.', + document: { paragraphs: [words(60, 'content')], rawText: `${words(60, 'content')} ${words(40, 'navigation')}` }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Very little paragraph content relative to total text receives a review suggestion.', + document: { paragraphs: [words(20, 'content')], rawText: `${words(20, 'content')} ${words(80, 'navigation')}` }, + expected: { score: 1, issues: 0, suggestions: 1 }, + }, + boundary: { + purpose: 'Exactly forty percent paragraph content stays at the documented middle threshold instead of the low band.', + document: { paragraphs: [words(40, 'content')], rawText: `${words(40, 'content')} ${words(60, 'navigation')}` }, + expected: { score: 3, issues: 0, suggestions: 0 }, + }, + }, + 'keyword-stuffing-detection': { + positive: { + purpose: 'Long content with diverse vocabulary receives the full repetition score.', + document: { rawText: words(80, 'term') }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Low-diversity language repeated across sentences triggers the stuffing heuristic.', + document: { rawText: stuffedText }, + expected: { score: 0, issues: 1, suggestions: 1 }, + }, + boundary: { + purpose: 'A short sample below fifty words is not classified from insufficient repetition evidence.', + document: { rawText: repeatedWords(49, 'widget') }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + }, + 'content-uniqueness-signals': { + positive: { + purpose: 'A verifiable original measurement plus a code example receives the full heuristic score.', + document: { rawText: 'Our research measured the documented fixture under controlled inputs.', html: '
npm run check
' }, + expected: { score: 5, issues: 0, suggestions: 0 }, + }, + negative: { + purpose: 'Generic prose with no original evidence or example remains at the base score.', + document: { rawText: 'The package checks content.', html: '

The package checks content.

' }, + expected: { score: 2, issues: 0, suggestions: 1 }, + }, + boundary: { + purpose: 'A code sample alone adds one point but cannot masquerade as original research.', + document: { rawText: 'Run the documented command.', html: '
aeoptimize --version
' }, + expected: { score: 3, issues: 0, suggestions: 1 }, + }, + }, +}; diff --git a/package.json b/package.json index 7aeba3d..9e062a7 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,12 @@ "dist/", "skills/", "agents/", + "fixtures/", + "examples/github-action-sample/", + "scripts/verify-release-v0.6.sh", ".claude-plugin/", "docs/methodology.md", + "docs/release-v0.6.md", "CHANGELOG.md", "CONTRIBUTING.md", "ROADMAP.md", diff --git a/scripts/verify-release-v0.6.sh b/scripts/verify-release-v0.6.sh new file mode 100755 index 0000000..338f657 --- /dev/null +++ b/scripts/verify-release-v0.6.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -u + +PACKAGE_NAME=aeoptimize +EXPECTED_VERSION=0.6.0 +EXPECTED_TAG=v0.6.0 +REPOSITORY=cucuwang/aeoptimize +EXPECTED_COMMIT=${1:-} +EXPECTED_PACKAGE_SHA256=${2:-} +EXPECTED_REPOSITORY_URL=git+https://github.com/cucuwang/aeoptimize.git +EXPECTED_HOMEPAGE=https://github.com/cucuwang/aeoptimize +EXPECTED_BUGS_URL=https://github.com/cucuwang/aeoptimize/issues + +if [ -z "$EXPECTED_COMMIT" ] || [ -z "$EXPECTED_PACKAGE_SHA256" ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +if ! [[ "$EXPECTED_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + echo "expected-release-commit must be a lowercase 40-character Git SHA" >&2 + exit 2 +fi + +if ! [[ "$EXPECTED_PACKAGE_SHA256" =~ ^[0-9a-f]{64}$ ]]; then + echo "expected-package-sha256 must be a lowercase 64-character SHA-256" >&2 + exit 2 +fi + +for command_name in awk curl jq npm git mktemp node; do + if ! command -v "$command_name" >/dev/null 2>&1; then + echo "missing required command: $command_name" >&2 + exit 2 + fi +done + +VERIFY_BASE=${TMPDIR:-/tmp} +VERIFY_BASE=${VERIFY_BASE%/} +VERIFY_ROOT=$(mktemp -d "$VERIFY_BASE/aeoptimize-release-verify.XXXXXX") +REGISTRY_JSON="$VERIFY_ROOT/registry.json" +RELEASE_JSON="$VERIFY_ROOT/release.json" +PACKAGE_TARBALL="$VERIFY_ROOT/$PACKAGE_NAME-$EXPECTED_VERSION.tgz" +CONSUMER_ROOT="$VERIFY_ROOT/consumer" +FAILURES=0 + +cleanup() { + if [ "${KEEP_VERIFY_ROOT:-0}" = "1" ]; then + echo "Verification workspace preserved: $VERIFY_ROOT" + return + fi + + case "$VERIFY_ROOT" in + "$VERIFY_BASE"/aeoptimize-release-verify.*) + rm -rf -- "$VERIFY_ROOT" + ;; + *) + echo "Refusing to remove unexpected verification path: $VERIFY_ROOT" >&2 + ;; + esac +} + +trap cleanup EXIT + +pass() { + echo "PASS: $1" +} + +note() { + echo "INFO: $1" +} + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +if curl -fsS "https://registry.npmjs.org/$PACKAGE_NAME" > "$REGISTRY_JSON"; then + latest=$(jq -r '."dist-tags".latest // empty' "$REGISTRY_JSON") + if [ "$latest" = "$EXPECTED_VERSION" ]; then + pass "npm latest is $EXPECTED_VERSION" + else + fail "npm latest is ${latest:-missing}; expected $EXPECTED_VERSION" + fi + + if jq -e --arg version "$EXPECTED_VERSION" '.versions[$version] != null' "$REGISTRY_JSON" >/dev/null; then + pass "npm contains exact version $EXPECTED_VERSION" + + published_git_head=$(jq -r --arg version "$EXPECTED_VERSION" '.versions[$version].gitHead // empty' "$REGISTRY_JSON") + if [ -z "$published_git_head" ]; then + note "npm does not expose gitHead; tarball SHA-256 remains the artifact identity gate" + elif [ "$published_git_head" = "$EXPECTED_COMMIT" ]; then + pass "npm gitHead matches $EXPECTED_COMMIT" + else + fail "npm gitHead is $published_git_head; expected $EXPECTED_COMMIT" + fi + + published_repository=$(jq -r --arg version "$EXPECTED_VERSION" '(.versions[$version].repository | if type == "object" then .url else . end) // empty' "$REGISTRY_JSON") + published_homepage=$(jq -r --arg version "$EXPECTED_VERSION" '.versions[$version].homepage // empty' "$REGISTRY_JSON") + published_bugs=$(jq -r --arg version "$EXPECTED_VERSION" '.versions[$version].bugs.url // empty' "$REGISTRY_JSON") + + if [ "$published_repository" = "$EXPECTED_REPOSITORY_URL" ] && \ + [ "$published_homepage" = "$EXPECTED_HOMEPAGE" ] && \ + [ "$published_bugs" = "$EXPECTED_BUGS_URL" ]; then + pass "npm repository identity matches $REPOSITORY" + else + fail "npm repository identity does not match $REPOSITORY" + fi + + tarball_url=$(jq -r --arg version "$EXPECTED_VERSION" '.versions[$version].dist.tarball // empty' "$REGISTRY_JSON") + if [ -n "$tarball_url" ] && curl -fLsS "$tarball_url" -o "$PACKAGE_TARBALL"; then + package_sha256=$(node -e "const crypto=require('node:crypto');const fs=require('node:fs');const path=process.argv[1];console.log(crypto.createHash('sha256').update(fs.readFileSync(path)).digest('hex'))" "$PACKAGE_TARBALL") + if [ "$package_sha256" = "$EXPECTED_PACKAGE_SHA256" ]; then + pass "npm tarball SHA-256 matches the verified candidate" + else + fail "npm tarball SHA-256 is ${package_sha256:-missing}; expected $EXPECTED_PACKAGE_SHA256" + fi + else + fail "npm tarball could not be downloaded for SHA-256 verification" + fi + + if npm --cache "$VERIFY_ROOT/npm-cache" install \ + --ignore-scripts --no-audit --no-fund \ + --prefix "$CONSUMER_ROOT" "$PACKAGE_NAME@$EXPECTED_VERSION" >/dev/null; then + for binary in aeoptimize aeo aeo-cli; do + binary_version=$("$CONSUMER_ROOT/node_modules/.bin/$binary" --version 2>/dev/null || true) + if [ "$binary_version" = "$EXPECTED_VERSION" ]; then + pass "$binary resolves to $EXPECTED_VERSION from the public package" + else + fail "$binary returned ${binary_version:-no version}; expected $EXPECTED_VERSION" + fi + done + else + fail "clean consumer installation failed for $PACKAGE_NAME@$EXPECTED_VERSION" + fi + else + fail "npm does not contain exact version $EXPECTED_VERSION" + fi +else + fail "npm registry metadata could not be fetched" +fi + +tag_lines=$(git ls-remote --tags "https://github.com/$REPOSITORY.git" \ + "refs/tags/$EXPECTED_TAG" "refs/tags/$EXPECTED_TAG^{}" 2>/dev/null || true) +tag_commit=$(printf '%s\n' "$tag_lines" | awk -v peeled="refs/tags/$EXPECTED_TAG^{}" '$2 == peeled { print $1 }') +if [ -z "$tag_commit" ]; then + tag_commit=$(printf '%s\n' "$tag_lines" | awk -v direct="refs/tags/$EXPECTED_TAG" '$2 == direct { print $1 }') +fi + +if [ "$tag_commit" = "$EXPECTED_COMMIT" ]; then + pass "$EXPECTED_TAG points to $EXPECTED_COMMIT" +else + fail "$EXPECTED_TAG points to ${tag_commit:-missing}; expected $EXPECTED_COMMIT" +fi + +release_status=$(curl -sS -o "$RELEASE_JSON" -w '%{http_code}' \ + "https://api.github.com/repos/$REPOSITORY/releases/tags/$EXPECTED_TAG" || true) +if [ "$release_status" = "200" ]; then + release_state=$(jq -r '[.tag_name, (.draft | tostring), (.prerelease | tostring)] | @tsv' "$RELEASE_JSON") + if [ "$release_state" = "$EXPECTED_TAG"$'\tfalse\tfalse' ]; then + pass "GitHub Release is published for $EXPECTED_TAG" + else + fail "GitHub Release is not a published non-prerelease for $EXPECTED_TAG" + fi +else + fail "GitHub Release lookup returned HTTP ${release_status:-error}" +fi + +if [ "$FAILURES" -ne 0 ]; then + echo "$FAILURES release verification check(s) failed." >&2 + exit 1 +fi + +echo "All public release checks passed." diff --git a/src/core/__tests__/release-contract.test.ts b/src/core/__tests__/release-contract.test.ts new file mode 100644 index 0000000..49a39ec --- /dev/null +++ b/src/core/__tests__/release-contract.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from 'vitest'; +import { readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ruleFixtureCorpus, ruleFixtureCorpusVersion, type RuleFixtureKind } from '../../../fixtures/v0.6/rule-corpus.js'; +import { allRules } from '../rules.js'; +import { scan } from '../scanner.js'; +import type { ParsedDocument } from '../types.js'; + +const testDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = join(testDirectory, '../../..'); +const fixtureKinds: RuleFixtureKind[] = ['positive', 'negative', 'boundary']; + +function makeDocument(overrides: Partial): ParsedDocument { + return { + url: 'fixture://v0.6-rule-corpus', + title: 'v0.6 rule fixture', + headings: [], + paragraphs: [], + jsonLd: [], + metaTags: {}, + links: [], + rawText: '', + ...overrides, + }; +} + +describe('v0.6 public rule fixture corpus', () => { + const scoredRules = allRules.filter((rule) => rule.weight > 0); + + it('covers every scored rule and only scored rules', () => { + expect(Object.keys(ruleFixtureCorpus).sort()).toEqual(scoredRules.map((rule) => rule.id).sort()); + }); + + for (const rule of scoredRules) { + describe(rule.id, () => { + for (const kind of fixtureKinds) { + it(`${kind} fixture matches the versioned expectation`, () => { + const fixture = ruleFixtureCorpus[rule.id][kind]; + const result = rule.evaluate(makeDocument(fixture.document)); + + expect(fixture.purpose.length).toBeGreaterThan(20); + expect(result.maxScore).toBe(rule.weight); + expect(result.score).toBe(fixture.expected.score); + expect(result.issues).toHaveLength(fixture.expected.issues); + expect(result.suggestions).toHaveLength(fixture.expected.suggestions); + }); + } + }); + } + + it('keeps the corpus, package, methodology, and Action sample on the same release version', async () => { + const packageJson = JSON.parse(await readFile(join(repositoryRoot, 'package.json'), 'utf8')); + const methodology = await readFile(join(repositoryRoot, 'docs/methodology.md'), 'utf8'); + const sampleWorkflow = await readFile( + join(repositoryRoot, 'examples/github-action-sample/.github/workflows/aeoptimize.yml'), + 'utf8', + ); + + expect(ruleFixtureCorpusVersion).toBe(packageJson.version); + expect(methodology).toContain(`v${packageJson.version} scoring contract`); + expect(sampleWorkflow).toContain(`uses: cucuwang/aeoptimize@v${packageJson.version}`); + + for (const rule of allRules) { + expect(methodology).toContain(`\`${rule.id}\``); + } + }); +}); + +describe('v0.6 JSON automation contract', () => { + it('keeps the documented top-level and page-level fields stable', async () => { + const report = await scan({ + type: 'file', + path: join(repositoryRoot, 'examples/github-action-sample/site/index.html'), + }); + + expect(Object.keys(report).sort()).toEqual(['overall', 'pages', 'summary', 'timestamp']); + expect(Object.keys(report.overall).sort()).toEqual([ + 'aiMetadata', + 'citability', + 'contentDensity', + 'schema', + 'structure', + 'total', + ]); + expect(report.pages).toHaveLength(1); + expect(Object.keys(report.pages[0]).sort()).toEqual(['issues', 'scores', 'suggestions', 'title', 'url']); + expect(Object.keys(report.pages[0].scores).sort()).toEqual(Object.keys(report.overall).sort()); + expect(Number.isNaN(Date.parse(report.timestamp))).toBe(false); + }); + + it('ships release and rollback instructions with the package', async () => { + const packageJson = JSON.parse(await readFile(join(repositoryRoot, 'package.json'), 'utf8')); + const releaseGuide = await readFile(join(repositoryRoot, 'docs/release-v0.6.md'), 'utf8'); + const publicVerifier = await readFile(join(repositoryRoot, 'scripts/verify-release-v0.6.sh'), 'utf8'); + + expect(packageJson.files).toContain('docs/release-v0.6.md'); + expect(packageJson.files).toContain('fixtures/'); + expect(packageJson.files).toContain('examples/github-action-sample/'); + expect(packageJson.files).toContain('scripts/verify-release-v0.6.sh'); + expect(releaseGuide).toContain('## Rollback'); + expect(releaseGuide).toContain('npm dist-tag add aeoptimize@0.5.3 latest'); + expect(releaseGuide).toContain(''); + expect(publicVerifier).toContain('.gitHead'); + expect(publicVerifier).toContain('EXPECTED_REPOSITORY_URL'); + expect(publicVerifier).toContain('.dist.tarball'); + expect(publicVerifier).toContain('EXPECTED_PACKAGE_SHA256'); + }); +}); diff --git a/src/core/__tests__/release-verifier.test.ts b/src/core/__tests__/release-verifier.test.ts new file mode 100644 index 0000000..970e720 --- /dev/null +++ b/src/core/__tests__/release-verifier.test.ts @@ -0,0 +1,148 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const testDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = join(testDirectory, '../../..'); +const verifier = join(repositoryRoot, 'scripts/verify-release-v0.6.sh'); +const expectedCommit = '0123456789abcdef0123456789abcdef01234567'; +const tarballContent = 'verified aeoptimize v0.6.0 candidate'; +const expectedTarballHash = createHash('sha256').update(tarballContent).digest('hex'); + +interface CommandResult { + code: number | null; + stdout: string; + stderr: string; +} + +function runVerifier(mockBin: string, packageHash: string, npmGitHead = expectedCommit): Promise { + return new Promise((resolve, reject) => { + const child = spawn('bash', [verifier, expectedCommit, packageHash], { + env: { + ...process.env, + PATH: `${mockBin}:${process.env.PATH}`, + MOCK_NPM_GIT_HEAD: npmGitHead, + MOCK_TAG_COMMIT: expectedCommit, + MOCK_TARBALL_CONTENT: tarballContent, + }, + }); + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.on('error', reject); + child.on('close', (code) => resolve({ code, stdout, stderr })); + }); +} + +async function writeExecutable(path: string, contents: string): Promise { + await writeFile(path, contents, 'utf8'); + await chmod(path, 0o755); +} + +describe('v0.6 public release verifier', () => { + let testRoot: string; + let mockBin: string; + + beforeEach(async () => { + testRoot = await mkdtemp(join(tmpdir(), 'aeoptimize-release-verifier-test-')); + mockBin = join(testRoot, 'bin'); + await mkdir(mockBin); + + await writeExecutable(join(mockBin, 'curl'), `#!/usr/bin/env bash +set -euo pipefail +output_file= +url= +while [ "$#" -gt 0 ]; do + case "$1" in + -o) output_file=$2; shift 2 ;; + -w) shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done + +case "$url" in + https://registry.npmjs.org/aeoptimize) + printf '{"dist-tags":{"latest":"0.6.0"},"versions":{"0.6.0":{"gitHead":"%s","repository":{"url":"git+https://github.com/cucuwang/aeoptimize.git"},"homepage":"https://github.com/cucuwang/aeoptimize","bugs":{"url":"https://github.com/cucuwang/aeoptimize/issues"},"dist":{"tarball":"https://registry.npmjs.org/aeoptimize/-/aeoptimize-0.6.0.tgz"}}}}' "$MOCK_NPM_GIT_HEAD" + ;; + https://registry.npmjs.org/aeoptimize/-/aeoptimize-0.6.0.tgz) + printf '%s' "$MOCK_TARBALL_CONTENT" > "$output_file" + ;; + https://api.github.com/repos/cucuwang/aeoptimize/releases/tags/v0.6.0) + printf '{"tag_name":"v0.6.0","draft":false,"prerelease":false}' > "$output_file" + printf '200' + ;; + *) + printf 'unexpected curl URL: %s\n' "$url" >&2 + exit 22 + ;; +esac +`); + + await writeExecutable(join(mockBin, 'git'), `#!/usr/bin/env bash +set -euo pipefail +printf '%s\trefs/tags/v0.6.0\n' "$MOCK_TAG_COMMIT" +`); + + await writeExecutable(join(mockBin, 'npm'), `#!/usr/bin/env bash +set -euo pipefail +prefix= +while [ "$#" -gt 0 ]; do + case "$1" in + --prefix) prefix=$2; shift 2 ;; + *) shift ;; + esac +done +mkdir -p "$prefix/node_modules/.bin" +for binary in aeoptimize aeo aeo-cli; do + printf '#!/usr/bin/env bash\nprintf "0.6.0\\n"\n' > "$prefix/node_modules/.bin/$binary" + chmod +x "$prefix/node_modules/.bin/$binary" +done +`); + }); + + afterEach(async () => { + await rm(testRoot, { recursive: true, force: true }); + }); + + it('passes only when npm metadata, tarball, aliases, tag, and Release match', async () => { + const result = await runVerifier(mockBin, expectedTarballHash); + + expect(result.code).toBe(0); + expect(result.stderr).toBe(''); + expect(result.stdout).toContain('PASS: npm gitHead matches'); + expect(result.stdout).toContain('PASS: npm tarball SHA-256 matches the verified candidate'); + expect(result.stdout).toContain('All public release checks passed.'); + }); + + it('fails closed when npm serves a different tarball', async () => { + const differentHash = createHash('sha256').update('different candidate').digest('hex'); + const result = await runVerifier(mockBin, differentHash); + + expect(result.code).toBe(1); + expect(result.stderr).toContain('FAIL: npm tarball SHA-256 is'); + expect(result.stdout).not.toContain('All public release checks passed.'); + }); + + it('accepts a missing optional gitHead when the tarball identity matches', async () => { + const result = await runVerifier(mockBin, expectedTarballHash, ''); + + expect(result.code).toBe(0); + expect(result.stdout).toContain('INFO: npm does not expose gitHead'); + expect(result.stdout).toContain('PASS: npm tarball SHA-256 matches the verified candidate'); + }); + + it('fails closed when npm exposes a different gitHead', async () => { + const differentCommit = 'fedcba9876543210fedcba9876543210fedcba98'; + const result = await runVerifier(mockBin, expectedTarballHash, differentCommit); + + expect(result.code).toBe(1); + expect(result.stderr).toContain(`FAIL: npm gitHead is ${differentCommit}`); + }); +});