diff --git a/.github/actions/download-frontend-artifact/action.yml b/.github/actions/download-frontend-artifact/action.yml index d86b0541b..c8f242a7a 100644 --- a/.github/actions/download-frontend-artifact/action.yml +++ b/.github/actions/download-frontend-artifact/action.yml @@ -8,9 +8,13 @@ description: >- inputs: branch: - description: Branch whose latest successful frontend build to fetch (main or develop). + # Always `main`, including for release builds: this action needs a *previously successful* + # run on the branch, and `release` only ever gets the commit that is already on `main`. + # Sourcing from `release` would fetch the previous release's SPA, or nothing at all on the + # first one. + description: Branch whose latest successful frontend build to fetch. required: false - default: develop + default: main runs: using: composite diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..46b5c74dd --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,36 @@ +# Categorises the notes GitHub generates for a release (`--generate-notes` in +# .github/workflows/github-release.yml). +# +# NOTE: GitHub categorises by PR **label**, not by PR-title prefix. The `feat:` / `fix:` / +# `docs:` / `chore:` title convention in extralit/docs/community/contributor.md is for humans +# and stays unenforced β€” to make a PR land under a heading below, give it the matching label. +# Anything unlabelled still shows up, under "Other changes". +changelog: + exclude: + authors: + # Exact logins β€” GitHub matches these verbatim, so the `[bot]` suffix is required. + - dependabot[bot] + - github-actions[bot] + categories: + - title: ✨ Features + labels: + - enhancement + - feature + - title: πŸ› Fixes + labels: + - bug + - fix + - title: πŸ“š Documentation + labels: + - documentation + - title: 🧹 Maintenance + labels: + - refactor + - refactoring + - infrastructure + - deployment + - chore + # Catch-all. Must stay last β€” GitHub assigns each PR to the first matching category. + - title: Other changes + labels: + - "*" diff --git a/.github/workflows/extralit-frontend.build-push-dev.yml b/.github/workflows/extralit-frontend.build-push-dev.yml index dc84e421a..8fc8c3fd9 100644 --- a/.github/workflows/extralit-frontend.build-push-dev.yml +++ b/.github/workflows/extralit-frontend.build-push-dev.yml @@ -6,7 +6,7 @@ name: Deploy PR preview HF Space # Deliberately decoupled from extralit-server.yml so it does NOT run on every push: # - auto: only when a PR (touching the server or frontend) is marked "ready for review" # - manual: workflow_dispatch with a PR number, to (re)deploy a preview on demand -# main/develop deploys keep flowing through extralit-server.yml as before. +# main/release deploys keep flowing through extralit-server.yml as before. on: # Note: `ready_for_review` only fires on a draftβ†’ready transition. A PR opened @@ -66,11 +66,9 @@ jobs: # No pytest here β€” preview-only path. The live frontend preview for this PR is published to # Vercel by extralit-frontend.yml; the HF Space only needs a bundled fallback UI, so bake - # the prebuilt develop frontend artifact instead of rebuilding from source. + # the prebuilt frontend artifact from `main` instead of rebuilding from source. - name: Download prebuilt frontend statics uses: ./.github/actions/download-frontend-artifact - with: - branch: develop - name: Build package run: | @@ -102,7 +100,7 @@ jobs: secrets: inherit # Opportunistic: this PR has its own ephemeral HF Space (extralit-dev/pr-), so point the - # PR's Vercel preview at it instead of the shared develop backend. Vercel's native Git + # PR's Vercel preview at it instead of the shared dev backend. Vercel's native Git # integration builds the preview; we just set a branch-scoped Preview env var (read by # extralit-frontend/vercel.ts at build time) and redeploy the branch's latest preview so it # takes effect now. Best-effort β€” never blocks the preview pipeline. diff --git a/.github/workflows/extralit-frontend.yml b/.github/workflows/extralit-frontend.yml index 818359cd7..77b0841cb 100644 --- a/.github/workflows/extralit-frontend.yml +++ b/.github/workflows/extralit-frontend.yml @@ -10,9 +10,12 @@ on: push: branches: - main - - develop - - feat/** - - releases/** + - release + paths: + - "extralit-frontend/**" + - ".github/workflows/extralit-frontend.yml" + + pull_request: paths: - "extralit-frontend/**" - ".github/workflows/extralit-frontend.yml" diff --git a/.github/workflows/extralit-server.build-docker-images.yml b/.github/workflows/extralit-server.build-docker-images.yml index 83eb94293..63a9ef879 100644 --- a/.github/workflows/extralit-server.build-docker-images.yml +++ b/.github/workflows/extralit-server.build-docker-images.yml @@ -32,9 +32,8 @@ jobs: - name: Read package info id: package-info - working-directory: extralit-server run: | - PACKAGE_VERSION=$(grep '__version__' src/extralit_server/_version.py | cut -d'"' -f2) + PACKAGE_VERSION=$(python3 scripts/bump_version.py check) PACKAGE_NAME="extralit-server" echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_OUTPUT echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/extralit-server.yml b/.github/workflows/extralit-server.yml index 1b5f736f6..f22930691 100644 --- a/.github/workflows/extralit-server.yml +++ b/.github/workflows/extralit-server.yml @@ -1,7 +1,9 @@ name: Build Extralit server package concurrency: - group: ${{ github.workflow }}-${{ github.sha }} + # Keyed on the ref, not the sha: a release pushes `main`, `release` and the tag at the same + # commit, so a sha-keyed group would make those runs cancel each other at random. + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: @@ -10,9 +12,13 @@ on: push: branches: - main - - develop - - feat/** - - releases/** + - release + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + paths: + - "extralit-server/**" + + pull_request: paths: - "extralit-server/**" @@ -22,7 +28,6 @@ permissions: jobs: build: name: Build `extralit-server` package - if: github.event.pull_request.draft == false runs-on: ubuntu-latest defaults: @@ -128,11 +133,9 @@ jobs: # The server no longer builds the frontend from source. The live UI is published to Vercel # by extralit-frontend.yml; here we just bake a prebuilt SPA into the wheel as the bundled - # fallback UI (main for release builds, else develop). + # fallback UI. Always sourced from `main` (see the action for why). - name: Download prebuilt frontend statics uses: ./.github/actions/download-frontend-artifact - with: - branch: ${{ github.ref_name == 'main' && 'main' || 'develop' }} - name: Build package run: | @@ -152,28 +155,26 @@ jobs: build_docker_images: name: Build docker images uses: ./.github/workflows/extralit-server.build-docker-images.yml - if: | - github.ref == 'refs/heads/main' - || github.ref == 'refs/heads/develop' - || contains(github.ref, 'releases/') - || github.event_name == 'workflow_dispatch' - || (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft) + # Branch pushes (`main`, `release`) and manual dispatch only. PRs get tests here and nothing + # else β€” their preview images are built by extralit-frontend.build-push-dev.yml on + # `ready_for_review`. Tags only publish to PyPI; the image was already built from `release`. + if: github.ref_type == 'branch' && github.event_name != 'pull_request' needs: - build with: - is_release: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} - publish_latest: ${{ github.ref == 'refs/heads/main' }} + is_release: ${{ github.ref_name == 'release' }} + publish_latest: ${{ github.ref_name == 'release' }} secrets: inherit # This job will publish extralit-server python package into PyPI repository publish_release: name: Publish Release runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + # Tags are the only thing that publishes. release.yml creates them; never push one by hand. + if: startsWith(github.ref, 'refs/tags/v') needs: - build - - build_docker_images defaults: run: @@ -203,7 +204,7 @@ jobs: - name: Read package info run: | - PACKAGE_VERSION=$(grep '__version__' src/extralit_server/_version.py | cut -d'"' -f2) + PACKAGE_VERSION=$(python3 "$GITHUB_WORKSPACE/scripts/bump_version.py" check) PACKAGE_NAME="extralit-server" echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV @@ -220,6 +221,5 @@ jobs: pip install --index-url https://test.pypi.org/simple --no-deps $PACKAGE_NAME==$PACKAGE_VERSION - name: Publish Package to PyPI πŸ₯© - if: github.ref == 'refs/heads/main' run: | uv publish --token ${{ secrets.AR_PYPI_API_TOKEN }} dist/* diff --git a/.github/workflows/extralit.docs.yml b/.github/workflows/extralit.docs.yml index 820d061fe..bb395c865 100644 --- a/.github/workflows/extralit.docs.yml +++ b/.github/workflows/extralit.docs.yml @@ -12,7 +12,6 @@ on: - "v[0-9]+.[0-9]+.[0-9]+" branches: - "main" - - "develop" - "docs/**" paths: - ".github/workflows/extralit.docs.yml" @@ -63,25 +62,35 @@ jobs: git config --global user.email "${{ github.actor }}@users.noreply.github.com" - name: Print GitHub ref info - run: echo "${{ github.ref }}" - echo "${{ github.head_ref }}" + env: + REF: ${{ github.ref }} + HEAD_REF: ${{ github.head_ref }} + run: echo "ref=$REF head_ref=$HEAD_REF" + # Trunk is the bleeding edge, so `main` is what you get by default; `stable` is moved by + # the tag step below. (Before the trunk-based migration this was inverted: `main` was + # `stable` and `develop` was `latest`.) - name: Deploy Extralit docs (branch /main) - run: | - uv run mike deploy stable --push - if: github.ref == 'refs/heads/main' - - - name: Deploy Extralit docs (branch /develop) run: | uv run mike deploy latest --push uv run mike set-default --push latest - if: github.ref == 'refs/heads/develop' || github.event_name == 'workflow_dispatch' + if: github.ref == 'refs/heads/main' - name: Deploy Extralit docs (release $version) run: | version=$(echo $TAG_VERSION | awk -F \. {'print $1"."$2'}) echo "Deploying version ${version}" - uv run mike deploy $version --push + # One-time migration, then a permanent no-op: the pre-trunk workflow published `stable` + # as its own *version* (a duplicate copy of the docs). mike refuses an alias whose name + # collides with a version, and --update-aliases only re-points names that are already + # aliases β€” so without this the first tagged release fails outright. Once `stable` is an + # alias it no longer appears as a version and this matches nothing. + if uv run mike list --json | jq -e '.[] | select(.version == "stable")' > /dev/null; then + echo "Retiring the legacy 'stable' version so it can become an alias" + uv run mike delete stable --push + fi + # `stable` as an alias of vX.Y, --update-aliases so it moves off the previous release. + uv run mike deploy "$version" stable --update-aliases --push if: startsWith(github.ref, 'refs/tags/') env: TAG_VERSION: ${{ github.ref_name }} diff --git a/.github/workflows/extralit.yml b/.github/workflows/extralit.yml index 817d199f8..b52230ee7 100644 --- a/.github/workflows/extralit.yml +++ b/.github/workflows/extralit.yml @@ -1,7 +1,9 @@ name: Build Extralit Package & Publish concurrency: - group: ${{ github.workflow }}-${{ github.sha }} + # Keyed on the ref, not the sha: a release pushes `main`, `release` and the tag at the same + # commit, so a sha-keyed group would make those runs cancel each other at random. + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: @@ -10,9 +12,15 @@ on: push: branches: - main - - develop - - feat/** - - releases/** + - release + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + paths: + - "extralit/**" + - "!extralit/docs/**" + - "!extralit/mkdocs.yml" + + pull_request: paths: - "extralit/**" - "!extralit/docs/**" @@ -24,7 +32,6 @@ permissions: jobs: build: - if: github.event.pull_request.draft == false services: extralit-container: image: extralitdev/extralit-hf-space:latest @@ -133,7 +140,8 @@ jobs: publish_release: name: Publish Release runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} + # Tags are the only thing that publishes. release.yml creates them; never push one by hand. + if: startsWith(github.ref, 'refs/tags/v') permissions: # This permission is needed for private repositories. @@ -167,7 +175,7 @@ jobs: - name: Read package info run: | - PACKAGE_VERSION=$(grep '__version__' src/extralit/_version.py | cut -d'"' -f2) + PACKAGE_VERSION=$(python3 "$GITHUB_WORKSPACE/scripts/bump_version.py" check) PACKAGE_NAME="extralit" echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV @@ -184,6 +192,5 @@ jobs: pip3 install --index-url https://test.pypi.org/simple --no-deps $PACKAGE_NAME==$PACKAGE_VERSION - name: Publish Package to PyPI πŸ₯© - if: github.ref == 'refs/heads/main' run: | uv publish --token ${{ secrets.AR_PYPI_API_TOKEN }} dist/* diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml new file mode 100644 index 000000000..5da686d82 --- /dev/null +++ b/.github/workflows/github-release.yml @@ -0,0 +1,70 @@ +name: GitHub Release + +# Creates the GitHub Release for a vX.Y.Z tag, but only once PyPI actually serves that version β€” +# a release note that links to an uninstallable version is worse than a late one. +# +# This workflow deliberately runs ZERO project code: no checkout, no install, no build. It holds +# `contents: write` and is triggered by a tag, so a malicious tagged commit must have nothing here +# to execute. Everything it needs (the tag, .github/release.yml) is read server-side by `gh`. + +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + +permissions: + contents: write + +concurrency: + group: github-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + name: Create GitHub Release + runs-on: ubuntu-latest + # PyPI publishing is driven by this same tag push, so the wait below is for a sibling run, + # not a finished one. Generous, because extralit-server publishes behind a full test suite. + timeout-minutes: 90 + steps: + - name: Wait for PyPI to serve this version + env: + VERSION: ${{ github.ref_name }} + run: | + set -euo pipefail + version="${VERSION#v}" + for pkg in extralit extralit-server; do + echo "::group::Waiting for $pkg==$version" + for attempt in $(seq 1 60); do + if curl -fsS -o /dev/null "https://pypi.org/pypi/$pkg/$version/json"; then + echo "$pkg==$version is live on PyPI (attempt $attempt)." + break + fi + if [[ "$attempt" -eq 60 ]]; then + echo "::endgroup::" + echo "::error::$pkg==$version never appeared on PyPI after 30 min. \ + Check the publish job on this tag; the GitHub Release is deliberately not created." + exit 1 + fi + sleep 30 + done + echo "::endgroup::" + done + + - name: Create the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + if gh release view "$TAG" --repo "$REPO" > /dev/null 2>&1; then + echo "Release $TAG already exists; nothing to do." + exit 0 + fi + # --verify-tag: `gh` refuses to mint a tag of its own, so a release can only ever + # describe a tag that release.yml already pushed. + # Published, not drafted: `-f dry_run=false` on release.yml WAS the human decision. + # Add --draft here if you'd rather curate the notes before they go public. + gh release create "$TAG" --repo "$REPO" --verify-tag --generate-notes --title "$TAG" + echo "Created https://github.com/$REPO/releases/tag/$TAG" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..9dc77b16d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,317 @@ +name: Release + +# The whole release, in one dispatch: +# +# gh workflow run release.yml -f version=X.Y.Z # dry run (the default): plan only +# gh workflow run release.yml -f version=X.Y.Z -f dry_run=false # cut it +# +# `cut` stamps the version and pushes `main`, `release` and `vX.Y.Z` in ONE atomic push, so all +# three always agree. Everything downstream is driven by those refs and lives in other workflows: +# `release` β†’ production image β†’ extralit/public-demo; the tag β†’ PyPI, versioned docs, GitHub Release. +# +# Re-running with the same version is safe: `plan` detects a converged release and `cut` no-ops. + +on: + workflow_dispatch: + inputs: + version: + description: "X.Y.Z (no leading v)" + required: true + type: string + dry_run: + description: "Plan only β€” push nothing" + type: boolean + default: true + skip_ci_check: + description: "Skip the green-CI assertion (use deliberately)" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + # Never two releases at once, and never cancel one halfway through its atomic push. + group: release + cancel-in-progress: false + +jobs: + authorize: + name: Authorize + # Forks are inert: a fork's dispatch can't move this repo's refs, so don't pretend to try. + if: github.repository == 'Extralit/extralit' + runs-on: ubuntu-latest + steps: + - name: Require admin or maintain + env: + # The PAT can always read collaborator permissions; GITHUB_TOKEN may not, depending on + # repo settings. Falls back so a fresh checkout of this repo still works. + GH_TOKEN: ${{ secrets.GH_ACTIONS_REPOSITORY_DISPATCH || github.token }} + REPO: ${{ github.repository }} + ACTOR: ${{ github.actor }} + run: | + set -euo pipefail + role=$(gh api "repos/$REPO/collaborators/$ACTOR/permission" --jq .role_name) + echo "$ACTOR has role '$role' on $REPO" + case "$role" in + admin | maintain) ;; + *) + # Leads with the phrase release_guide.md tells operators to look for. + echo "::error::Not authorized: '$ACTOR' needs admin or maintain to cut a release (has '$role')." + exit 1 + ;; + esac + + plan: + name: Plan + needs: authorize + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + outputs: + version: ${{ steps.validate.outputs.version }} + tag: ${{ steps.validate.outputs.tag }} + base_sha: ${{ steps.base.outputs.sha }} + already_done: ${{ steps.converged.outputs.already_done }} + release_sha: ${{ steps.base.outputs.release_sha }} + steps: + - name: Validate version + id: validate + env: + # Never interpolate ${{ inputs.* }} straight into a run: body β€” bind it to env first. + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + # Deliberately the same shape scripts/bump_version.py accepts, so the two can't disagree. + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::version must be X.Y.Z with no leading 'v' (got '$VERSION')" + exit 1 + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag=v$VERSION" >> "$GITHUB_OUTPUT" + + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + # Full history + tags: the convergence check reads out an existing tag's commit. + fetch-depth: 0 + # This job only reads. Nothing here needs a credential in .git/config. + persist-credentials: false + + - name: Resolve base commit and the current release pointer + id: base + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + # The lease value for `cut`'s push. Empty when `release` does not exist yet (first + # release) β€” git reads an empty as "the ref must not already exist". Asked of + # the API rather than a local ref so it doesn't depend on what checkout happened to fetch. + release_sha=$(gh api "repos/$REPO/git/ref/heads/release" --jq .object.sha 2>/dev/null || true) + echo "Current refs/heads/release: ${release_sha:-}" + echo "release_sha=$release_sha" >> "$GITHUB_OUTPUT" + + - name: Check whether this release already happened + id: converged + env: + TAG: ${{ steps.validate.outputs.tag }} + VERSION: ${{ steps.validate.outputs.version }} + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + set -euo pipefail + if ! git rev-parse -q --verify "refs/tags/$TAG^{commit}" > /dev/null; then + echo "Tag $TAG does not exist yet β€” this is a fresh release." + echo "already_done=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # The tag exists. It's a converged re-run only if the commit it points at actually + # carries this version; anything else is a collision a human has to resolve. + # + # Read the version straight out of the tag's tree. Deliberately NOT `git checkout $TAG` + # + `bump_version.py`: that executes code from a commit this job did not vet, which is + # exactly what github-release.yml is designed to avoid. `|| true` because the step runs + # under `pipefail` and an old tag may predate one of these paths β€” a missing file must + # read as a mismatch, not abort the step. + at_tag() { git show "$TAG:$1" 2>/dev/null || true; } + py_version() { at_tag "$1" | sed -n 's/^__version__[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p'; } + + sdk=$(py_version extralit/src/extralit/_version.py) + server=$(py_version extralit-server/src/extralit_server/_version.py) + frontend=$(at_tag extralit-frontend/package.json | jq -r '.version // empty') + + if [[ "$sdk" == "$VERSION" && "$server" == "$VERSION" && "$frontend" == "$VERSION" ]]; then + echo "Tag $TAG already points at a commit stamped $VERSION β€” nothing to do." + echo "already_done=true" >> "$GITHUB_OUTPUT" + else + echo "::error::Tag $TAG already exists at a different commit \ + ($(git rev-parse --short "$TAG^{commit}") β€” sdk=${sdk:-?} server=${server:-?} frontend=${frontend:-?}, \ + expected $VERSION). If this is recovery, delete the tag first: git push origin :refs/tags/$TAG" + exit 1 + fi + + - name: Assert CI is green on the base commit + if: ${{ !inputs.skip_ci_check }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + BASE_SHA: ${{ steps.base.outputs.sha }} + run: | + set -euo pipefail + # Filter out this workflow, otherwise a dispatch from `main` waits on its own run forever. + runs=$(gh api "repos/$REPO/actions/runs?head_sha=$BASE_SHA&per_page=100" \ + --jq '[.workflow_runs[] | select(.name != "Release") | {name, status, conclusion}]') + + # All three branches lead with "CI is not green on main" β€” the phrase release_guide.md + # tells operators to search the log for β€” then say which of the three cases it is. + total=$(jq length <<< "$runs") + if [[ "$total" -eq 0 ]]; then + echo "::error::CI is not green on main ($BASE_SHA): no workflow runs found at all. \ + Nothing has validated this commit β€” wait for CI, or re-run with skip_ci_check=true if it \ + genuinely has no applicable workflows." + exit 1 + fi + + pending=$(jq -r '[.[] | select(.status != "completed") | .name] | join(", ")' <<< "$runs") + if [[ -n "$pending" ]]; then + echo "::error::CI is not green on main ($BASE_SHA): still running β€” $pending. \ + Wait for it to finish." + exit 1 + fi + + failed=$(jq -r '[.[] | select(.conclusion | IN("success","skipped","neutral") | not) | .name] | join(", ")' <<< "$runs") + if [[ -n "$failed" ]]; then + echo "::error::CI is not green on main ($BASE_SHA): failing β€” $failed. \ + Fix it, or re-run with skip_ci_check=true." + exit 1 + fi + + echo "All $total workflow run(s) on $BASE_SHA are green." + + - name: Write the plan + env: + VERSION: ${{ steps.validate.outputs.version }} + TAG: ${{ steps.validate.outputs.tag }} + BASE_SHA: ${{ steps.base.outputs.sha }} + ALREADY_DONE: ${{ steps.converged.outputs.already_done }} + DRY_RUN: ${{ inputs.dry_run }} + SKIP_CI_CHECK: ${{ inputs.skip_ci_check }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + if [[ "$DRY_RUN" == "true" ]]; then mode='**DRY RUN** β€” nothing will be pushed'; else mode='**EXECUTE**'; fi + + { + echo "## Release plan" + echo + echo "| | |" + echo "| --- | --- |" + echo "| Mode | $mode |" + echo "| Version | \`$VERSION\` |" + echo "| Tag | \`$TAG\` |" + echo "| Base commit | [\`${BASE_SHA:0:12}\`](https://github.com/$REPO/commit/$BASE_SHA) on \`main\` |" + echo "| Converged already | \`$ALREADY_DONE\` |" + echo "| CI check | $([[ "$SKIP_CI_CHECK" == "true" ]] && echo 'skipped by request' || echo 'green') |" + echo + if [[ "$ALREADY_DONE" == "true" ]]; then + echo "\`$TAG\` already points at a commit stamped \`$VERSION\`. Every job below is a no-op." + elif [[ "$DRY_RUN" == "true" ]]; then + echo "Re-run with \`-f dry_run=false\` to stamp \`$VERSION\` and push" + echo "\`main\`, \`release\` and \`$TAG\` atomically." + else + echo "Stamping \`$VERSION\` onto \`${BASE_SHA:0:12}\` and pushing" + echo "\`main\`, \`release\` and \`$TAG\` atomically." + fi + } >> "$GITHUB_STEP_SUMMARY" + + cut: + name: Cut + needs: [authorize, plan] + if: ${{ !inputs.dry_run && needs.plan.outputs.already_done != 'true' }} + runs-on: ubuntu-latest + steps: + - name: Checkout the base commit + uses: actions/checkout@v4 + with: + ref: ${{ needs.plan.outputs.base_sha }} + # A PAT (or App token), NOT GITHUB_TOKEN: pushes made with GITHUB_TOKEN emit no + # downstream `push` events, so the tag would never trigger PyPI, docs, or the Release. + token: ${{ secrets.GH_ACTIONS_REPOSITORY_DISPATCH }} + # The next step runs repository code (bump_version.py). This token pushes straight to + # `main`/`release`/tags past branch protection, so don't leave it in .git/config while + # that happens β€” the push step below supplies it explicitly, and only to itself. + persist-credentials: false + fetch-depth: 0 + + - name: Stamp the version + env: + VERSION: ${{ needs.plan.outputs.version }} + run: | + set -euo pipefail + python3 scripts/bump_version.py set --version "$VERSION" + python3 scripts/bump_version.py check --expect "$VERSION" + + - name: Commit and push main + release + tag + env: + VERSION: ${{ needs.plan.outputs.version }} + TAG: ${{ needs.plan.outputs.tag }} + # What `plan` saw at refs/heads/release; empty if the branch doesn't exist yet. + RELEASE_SHA: ${{ needs.plan.outputs.release_sha }} + # Scoped to this step alone β€” see persist-credentials above. Actions masks it in logs. + TOKEN: ${{ secrets.GH_ACTIONS_REPOSITORY_DISPATCH }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Empty when re-running after a partial failure that already stamped and committed. + if [[ -n "$(git status --porcelain)" ]]; then + git commit -aqm "release: $TAG" + else + echo "Already stamped at $VERSION; reusing $(git rev-parse --short HEAD)." + fi + git tag "$TAG" + + # --atomic: either all three refs move or none do. A stale `main` fails the whole push + # rather than leaving a tag with no branch (or worse, a release with no tag). + # + # `release` is a production *pointer*, not a branch that accumulates work, so it moves + # deliberately rather than by fast-forward β€” otherwise a rollback (release_guide.md + # force-moves it back to an earlier tag) can leave it off `main`'s ancestry and abort + # the next release *after* the version was already stamped and committed. The lease is + # scoped to this one ref, so `main` and the tag stay strictly non-forced; if anyone + # moved `release` since `plan` read it, the push is rejected instead of clobbering. + # An empty RELEASE_SHA means "must not already exist" β€” the first release. + git push --atomic \ + --force-with-lease=refs/heads/release:"$RELEASE_SHA" \ + "https://x-access-token:${TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + "HEAD:refs/heads/main" \ + "HEAD:refs/heads/release" \ + "refs/tags/$TAG" + + - name: What happens next + env: + TAG: ${{ needs.plan.outputs.tag }} + VERSION: ${{ needs.plan.outputs.version }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + { + echo "## Released \`$TAG\`" + echo + echo "\`main\`, \`release\` and \`$TAG\` now point at the same commit. Watch:" + echo + echo "- [Actions](https://github.com/$REPO/actions) β€” \`release\` builds the multi-arch" + echo " production image and restarts the public demo; \`$TAG\` publishes to PyPI," + echo " deploys the \`v${VERSION%.*}\` docs, and creates the GitHub Release." + echo "- β€” should report \`$VERSION\`." + echo "- and" + echo " " + echo + echo "To roll back: re-point \`release\` at the previous tag and restart the Space." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CLAUDE.md b/CLAUDE.md index 336513d57..6b00f2908 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,6 +74,38 @@ cd extralit && uv run pytest tests # SDK tests cd extralit-frontend && npm run test # Frontend tests ``` +## Branching + +Trunk-based. **`main` is the trunk and the default branch** β€” every change, code or docs, +branches from `main` and squash-merges back via PR. There is no `develop` branch and no +`releases/**` branches. + +| Ref | Role | Deploys to | +|---|---|---| +| `main` | trunk; every merged PR | `extralit-dev/develop` (dev HF Space) | +| `release` | long-lived production pointer, moved only by `release.yml` | `extralit/public-demo` | +| `vX.Y.Z` tag | the release itself | PyPI, versioned docs, GitHub Release | +| PR (non-fork) | preview | ephemeral `extralit-dev/pr-N` | + +Branch names: `feat/*`, `fix/*`, `docs/*`, short-lived. PR titles use the matching +`feat:` / `fix:` / `docs:` / `chore:` prefix. Note the generated release notes group PRs by +**label**, not by title prefix (see `.github/release.yml`) β€” label a PR to place it under a +heading; the title convention is for humans. + +**Never push to `release` or create tags by hand.** Releases are one dispatch: + +```bash +gh workflow run release.yml -f version=X.Y.Z # dry run (the default) +gh workflow run release.yml -f version=X.Y.Z -f dry_run=false # cut it +``` + +That stamps the version via `scripts/bump_version.py` and pushes `main`, `release`, and the +tag atomically. The version lives in three files β€” always change it with +`python scripts/bump_version.py set --version X.Y.Z`, never by hand. + +See `docs/architecture/deployment.md` for the full pipeline and +`extralit/docs/community/release_guide.md` for the release runbook. + ## Git Workgrees When creating a git workgree, place it at `.worktree/` relative to the repo root, normalizing `/` to `-` in the branch-name. diff --git a/README.md b/README.md index 0b9411142..f684acd01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Extralit + Extralit

diff --git a/docs/architecture/deployment.md b/docs/architecture/deployment.md index 14ff4cd1f..2dee9592b 100644 --- a/docs/architecture/deployment.md +++ b/docs/architecture/deployment.md @@ -11,6 +11,32 @@ here as a git submodule). --- +## 0. Branching model + +Extralit is **trunk-based**. There is no `develop` branch and no `releases/**` +branches. + +| Ref | Role | Images | Deploys to | +| --- | --- | --- | --- | +| `main` | trunk, default branch; every merged PR | `extralitdev/*:main` + `:latest`, amd64 | `extralit-dev/develop` | +| `release` | long-lived production pointer | `extralit/*:vX.Y.Z` + `:latest`, amd64+arm64 | `extralit/public-demo` | +| `vX.Y.Z` tag | the release itself | β€” | PyPI, versioned docs, GitHub Release | +| PR (`N/merge`) | preview | `extralitdev/*:pr-N`, amd64 | ephemeral `extralit-dev/pr-N` | + +`release` is always a tagged point on `main`'s history: the release workflow +pushes one version-stamp commit to `main`, `release`, and the tag **atomically**, +so all three land at the same SHA. Nothing else ever writes to `release`, and +production only moves when [`release.yml`](../../.github/workflows/release.yml) +is deliberately dispatched. See +[`extralit/docs/community/release_guide.md`](../../extralit/docs/community/release_guide.md). + +**`is_release` is the only production signal.** Branch names are not load-bearing +in the deploy path β€” the HF Space repo routes on the dispatch payload's +`is_release` flag, so a payload without it can never reach production whatever +branch it names. + +--- + ## 1. The big picture The pipeline spans **two repositories** and is glued together by a GitHub @@ -19,12 +45,13 @@ The pipeline spans **two repositories** and is glued together by a GitHub ```mermaid flowchart TD subgraph monorepo["Repo: extralit/extralit (monorepo)"] - push["git push / merge"] + push["push to main / release"] + rel[".github/workflows/release.yml (manual dispatch)"] prready["PR ready_for_review / manual dispatch"] fe[".github/workflows/extralit-frontend.yml"] sdk[".github/workflows/extralit.yml"] srv[".github/workflows/extralit-server.yml"] - prev[".github/workflows/extralit-pr-preview.yml"] + prev[".github/workflows/extralit-frontend.build-push-dev.yml"] srvbuild["extralit-server.build-docker-images.yml"] end @@ -39,92 +66,128 @@ flowchart TD subgraph hf["Hugging Face"] demo["Space: extralit-public-demo.hf.space"] + dev["Space: extralit-dev-develop.hf.space"] prspace["PR preview Space: extralit-dev/pr-N"] end + rel -->|"atomic push: main + release + tag"| push push --> fe push --> sdk push --> srv prready -->|server OR frontend PR| prev - srv -->|tests + builds frontend dist + wheel| srvbuild - prev -->|builds frontend dist + wheel| srvbuild + fe -->|SPA artifact| srv + srv -->|tests + downloads SPA + wheel| srvbuild + prev -->|downloads SPA + wheel| srvbuild srvbuild -->|build & push| srvimg srvbuild -->|repository_dispatch: build-hf-space| bhs bhs -->|FROM server image + ES/Redis/OCR| spaceimg - bhs -->|restart| demo + bhs -->|"restart (is_release=true)"| demo + bhs -->|"restart (branch=main)"| dev bhs -->|duplicate + retarget| prspace ``` **Key insight:** the HF Space image is built **`FROM` the `extralit-server` -image**. The server image, in turn, bundles the **compiled frontend** at build -time. So the demo is rebuilt by whichever workflow rebuilds that server image: -the **`extralit-server`** workflow drives `main`/`develop`, and a dedicated -**`extralit-pr-preview`** workflow drives per-PR ephemeral Spaces (it rebuilds the -same server image β€” frontend baked in β€” so frontend-only PRs get a preview too). -Both hand off to the same `build-docker-images.yml` + `repository_dispatch`. See -[Β§6 Gotchas](#6-gotchas--operational-notes). +image**, and the server image bundles a **prebuilt frontend SPA** as its fallback +UI. So the demo is rebuilt by whichever workflow rebuilds that server image: the +**`extralit-server`** workflow drives `main` and `release`, and +**`extralit-frontend.build-push-dev.yml`** drives per-PR ephemeral Spaces (it +rebuilds the same server image β€” SPA baked in β€” so frontend-only PRs get a +preview too). Both hand off to the same `build-docker-images.yml` + +`repository_dispatch`. See [Β§6 Gotchas](#6-gotchas--operational-notes). --- ## 2. Module β†’ workflow routing Each module has its own workflow, gated by `paths:` filters so unrelated changes -don't trigger unrelated builds. +don't trigger unrelated builds. All three run on **push to `main`/`release`** and +on **`pull_request`**. | Module (dir) | Workflow | Triggers on path | Produces | Reaches the demo? | | ------------------- | --------------------------------- | --------------------------------------------- | --------------------------------------------------- | ----------------- | -| `extralit-server/` | `extralit-server.yml` | `extralit-server/**` (push/PR) | `extralit-server` Docker image + PyPI wheel | **Yes** (`main`/`develop`) | -| `extralit-frontend/`| `extralit-frontend.yml` | `extralit-frontend/**` | Frontend `dist/` artifact (tests/lint only) | IndirectΒΉ | -| *(server **or** frontend PR)* | `extralit-pr-preview.yml` | PR `ready_for_review` on `extralit-server/**` **or** `extralit-frontend/**`; or manual `pr_number` | `extralitdev/extralit-server:pr-N` image β†’ ephemeral Space | **Yes** (`extralit-dev/pr-N`) | +| `extralit-server/` | `extralit-server.yml` | `extralit-server/**` (push/PR) | `extralit-server` Docker image + PyPI wheel | **Yes** (`main` β†’ dev Space; `release` β†’ public demo) | +| `extralit-frontend/`| `extralit-frontend.yml` | `extralit-frontend/**` | Prerendered SPA artifact (plus tests/lint) | IndirectΒΉ | +| *(server **or** frontend PR)* | `extralit-frontend.build-push-dev.yml` | **Non-fork** PR `ready_for_review` on `extralit-server/**` **or** `extralit-frontend/**`; or manual `pr_number` (works for forks too) | `extralitdev/extralit-server:pr-N` image β†’ ephemeral Space | **Yes** (`extralit-dev/pr-N`) | | `extralit/` (SDK) | `extralit.yml` | `extralit/**` (excl. `docs/`, `mkdocs.yml`) | `extralit` PyPI wheel | NoΒ² | +| `extralit/docs/` | `extralit.docs.yml` | `extralit/docs/**`, `mkdocs.yml` | Versioned docs via `mike` β†’ `gh-pages` | No | +| *(whole repo)* | `release.yml` | manual dispatch only | version stamp on `main` + `release` + `vX.Y.Z` tag | **Yes** (indirectly) | +| *(none)* | `github-release.yml` | `vX.Y.Z` tag | GitHub Release, once PyPI serves both packages | NoΒ³ | | `extralit-hf-space/`| `build-hf-space.yml` *(other repo)* | repository_dispatch / manual | `extralit-hf-space` Docker image + Space restart/deploy | **Yes** (terminal) | -ΒΉ The frontend's own workflow only tests and uploads a `dist` artifact. The -frontend that actually ships is **recompiled from source inside the server -build** (see Β§3). A frontend-only change does not redeploy the **public** demo on -its own β€” that still needs the server/`main` flow β€” but a frontend-only **PR** -*does* get an ephemeral preview: `extralit-pr-preview.yml` rebuilds the server -image (frontend baked in) and deploys `extralit-dev/pr-N` (see Β§3a). +ΒΉ The frontend's own workflow tests, lints, and uploads a prerendered SPA +artifact; the **live** UI is deployed separately by Vercel's native Git +integration (configured in `extralit-frontend/vercel.ts`, not by any workflow). +The SPA the *server* ships is downloaded from this workflow's artifact (see Β§3). +A frontend-only change does not redeploy the public demo on its own β€” that needs +a release β€” but a frontend-only **PR** *does* get an ephemeral preview (see Β§3a). Β² The SDK is published to PyPI and is *consumed by* the demo's CI for integration tests (it spins up `extralitdev/extralit-hf-space:latest` as a service container), but SDK changes never rebuild the Space image. +Β³ [`github-release.yml`](../../.github/workflows/github-release.yml) runs **zero +project code** β€” no checkout, no install, no build. It holds `contents: write` +and fires on a tag, so a malicious tagged commit must have nothing there to +execute. It polls PyPI for both packages first and only then runs `gh release +create --verify-tag --generate-notes`, so a release is never announced before it +is installable and `gh` can never mint a tag of its own. Note headings come from +PR **labels** via [`.github/release.yml`](../../.github/release.yml) β€” the +`feat:`/`fix:` title convention is for humans and is not what GitHub reads. + --- ## 3. Stage A β€” `extralit-server.yml` (in the monorepo) File: [`.github/workflows/extralit-server.yml`](../../.github/workflows/extralit-server.yml) -Triggered by push to `main`/`develop`/`releases/**`, by non-fork PRs, or -manually (`workflow_dispatch`). +Triggered by push to `main`/`release`, by pull requests, or manually +(`workflow_dispatch`). ### Job `build` 1. Spins up service containers (Elasticsearch, Postgres, Redis, MinIO) and runs `pytest tests/unit` with coverage β†’ Codecov. -2. **Compiles the frontend from source** (`npm install && npm run build` in - `extralit-frontend/`) using `BASE_URL=@@baseUrl@@` (a placeholder rewritten at - runtime to support a parameterizable root path). -3. Copies `extralit-frontend/dist` into `src/extralit_server/static`, then - `uv build` β†’ uploads the `extralit-server` wheel artifact. +2. **Downloads the prebuilt frontend SPA** via the + [`download-frontend-artifact`](../../.github/actions/download-frontend-artifact) + composite action, which fetches the latest successful `extralit-frontend.yml` + artifact for `main` β€” **always `main`**, release builds included, because the + action needs a *previously successful* run on the branch it names and `release` + only ever carries a commit that already landed on `main`. The server does + **not** run `npm`. +3. Copies `extralit-frontend/.output/public` into `src/extralit_server/static`, + asserts `index.html` exists, then `uv build` β†’ uploads the `extralit-server` + wheel artifact. + +> **Why prebuilt, and why `BASE_URL=/`.** Since the Nuxt 4 migration the SPA is +> prerendered by `nuxi generate` at `BASE_URL=/` and shipped as static files; the +> old Nuxt 2 `@@baseUrl@@` placeholder breaks Nuxt 4's prerender crawler. +> Parameterizable sub-path hosting is a separate follow-up. The SPA is otherwise +> env-agnostic (it calls a relative `/api`), so one artifact works wherever the +> server is deployed. ### Job `build_docker_images` β†’ `extralit-server.build-docker-images.yml` File: [`.github/workflows/extralit-server.build-docker-images.yml`](../../.github/workflows/extralit-server.build-docker-images.yml) -Runs only on `main` / `develop` / `releases/**` / `workflow_dispatch` / non-fork -non-draft PRs. - -| Input | `is_release` (main / dispatch) | dev (develop / PR) | -| ---------------------- | ------------------------------ | ------------------------------- | -| Docker org | `extralit/extralit-server` | `extralitdev/extralit-server` | -| Platforms | `linux/amd64,linux/arm64` | `linux/amd64` | -| Image tag | `v` | branch name (cleaned) or `pr-N` | -| Publish `:latest` | only on `main` | dev `:latest`, except PR previews (`pr_number` set) | - -The tag is derived by the +Runs on branch pushes (`main` / `release`) and `workflow_dispatch` only β€” +`github.ref_type == 'branch' && github.event_name != 'pull_request'`. **PRs run +tests and nothing else here;** their preview images come from +`extralit-frontend.build-push-dev.yml` (Β§3a), which is the only caller that +passes `pr_number`. Tag pushes are excluded too: the image was already built from +the `release` push at the same SHA, and a tag-triggered build would dispatch +`branch=vX.Y.Z`, which `resolve-env` would turn into a junk preview Space. + +| Input | `is_release` (`release` branch) | dev (`main` / PR) | +| ---------------------- | ------------------------------- | ------------------------------- | +| Docker org | `extralit/extralit-server` | `extralitdev/extralit-server` | +| Platforms | `linux/amd64,linux/arm64` | `linux/amd64` | +| Image tag | `v` | branch name (cleaned) or `pr-N` | +| Publish `:latest` | only on `release` | dev `:latest`, except PR previews (`pr_number` set) | + +The version comes from +[`scripts/bump_version.py check`](../../scripts/bump_version.py), the single +owner of the three files that carry it. The dev tag is derived by the [`docker-image-tag-from-ref`](../../.github/actions/docker-image-tag-from-ref) -composite action: tags β†’ ``, PRs β†’ `pr-`, branches β†’ -`` with non-alphanumerics replaced by `-`. +composite action: tags β†’ ``, PRs β†’ `pr-`, branches β†’ `` with +non-alphanumerics replaced by `-`. After building & pushing the server image, the final step fires the cross-repo trigger: @@ -144,18 +207,21 @@ repo. `DISPATCH_BRANCH` is `github.ref_name` normally, or `/merge` when the reusable build is called with the optional `pr_number` input (manual PR preview), so `resolve-env` routes to `pr_space_slug=pr-N`. -> The `publish_release` job additionally publishes the `extralit-server` wheel to -> (Test)PyPI on `main`/dispatch. Not part of the Space deploy. +> The `publish_release` job publishes the `extralit-server` wheel to (Test)PyPI, +> and fires **only on a `vX.Y.Z` tag push** β€” not on branch pushes. Under +> trunk-based development a branch-gated publish would try to release on every +> merge to `main`. Not part of the Space deploy. --- -## 3a. PR preview β€” `extralit-pr-preview.yml` (in the monorepo) +## 3a. PR preview β€” `extralit-frontend.build-push-dev.yml` (in the monorepo) -File: [`.github/workflows/extralit-pr-preview.yml`](../../.github/workflows/extralit-pr-preview.yml) +File: [`.github/workflows/extralit-frontend.build-push-dev.yml`](../../.github/workflows/extralit-frontend.build-push-dev.yml) +(its `name:` is **"Deploy PR preview HF Space"**) -A single-purpose workflow that gives any server- **or** frontend-touching PR an -ephemeral Space, deliberately decoupled from `extralit-server.yml` so it does -**not** run on every push: +A single-purpose workflow that gives a server- **or** frontend-touching PR from a +branch **in this repository** an ephemeral Space, deliberately decoupled from +`extralit-server.yml` so it does **not** run on every push: - **Auto:** `pull_request: types: [ready_for_review]` on `extralit-server/**` or `extralit-frontend/**`. Marking a PR ready (not each subsequent push) builds the @@ -163,14 +229,27 @@ ephemeral Space, deliberately decoupled from `extralit-server.yml` so it does draftβ†’ready transition β€” a PR opened directly as non-draft won't auto-trigger; re-mark it ready or use the manual path.) - **Manual:** `workflow_dispatch` with a `pr_number` input β€” (re)deploy a chosen PR - on demand. - -Its `build` job mirrors the frontend+wheel steps from Β§3 (no pytest), then calls -the same [`build-docker-images.yml`](../../.github/workflows/extralit-server.build-docker-images.yml) + on demand, resolving `refs/pull//merge` so the run is self-contained. + +> **Fork PRs get no automatic preview.** Both guarded jobs carry +> `github.event.pull_request.head.repo.fork == false` (`build` at :43, +> `point-preview-at-pr-space` at :110; `build_docker_images` is skipped +> transitively via `needs`), because a fork PR receives no repository secrets β€” +> the Docker login and cross-repo dispatch would fail. A maintainer *can* still +> produce one: the **manual** path resolves `refs/pull//merge`, which exists for +> fork PRs, and runs in this repo's context with secrets. Review a fork PR's code +> before dispatching β€” the preview builds that PR's merge ref. + +Its `build` job downloads the prebuilt SPA and builds the wheel (no pytest), then +calls the same [`build-docker-images.yml`](../../.github/workflows/extralit-server.build-docker-images.yml) with `is_release: false` and `pr_number`. Result: `extralitdev/extralit-server:pr-N` is pushed and the dispatch (`branch=/merge`) drives the HF Space repo's `deploy-pr-space` job β†’ **`extralit-dev/pr-N`** (`extralit-dev-pr-N.hf.space`). -`main`/`develop` deploys are untouched and still flow through Β§3. +`main`/`release` deploys are untouched and still flow through Β§3. + +A third job, `point-preview-at-pr-space`, best-effort points the PR's **Vercel** +preview at that PR's own Space by setting a branch-scoped `API_BASE_URL` preview +env var and redeploying. It never blocks the pipeline. --- @@ -180,22 +259,29 @@ File: [`extralit-hf-space/.github/workflows/build-hf-space.yml`](../../extralit- Entry points: - **`repository_dispatch` / `build-hf-space`** β€” the automated path from Stage A. -- **`workflow_dispatch`** β€” manual rebuild of the current ref (`develop` env, tag - `latest`, amd64 only). +- **`workflow_dispatch`** β€” manual rebuild of the current ref. Carries no + payload, so it always resolves to `staging`; a manual run **cannot** reach + production. ### Job `resolve-env` -Pure-bash step that maps the dispatch payload to build parameters: +Pure-bash step that maps the dispatch payload to build parameters. It branches on +`is_release` **first**; branch names only distinguish trunk from preview: + +| Payload | `env_name` | `image_tag` | `tag_latest` | `platforms` | `pr_space_slug` | +| ------------------------------------- | ------------ | -------------------- | ------------ | ------------- | --------------- | +| `is_release=true` | `production` | payload `tag` (`v…`) | `true` | `amd64,arm64` | β€” (empty) | +| `branch=main` (or `develop`†) | `staging` | payload `tag` | `true` | `amd64` | β€” (empty) | +| any other branch (PR merge ref) | `staging` | payload `tag` | `false` | `amd64` | `pr-N` / slug | +| `workflow_dispatch` | `staging` | `latest` | per ref | `amd64` | per ref | -| Source branch / event | `env_name` | `image_tag` | `tag_latest` | `platforms` | `pr_space_slug` | -| ------------------------------------- | ---------- | ------------------ | ------------ | --------------------------- | --------------- | -| `is_release=true` β†’ `main` | `main` | payload `tag` (`v…`) | `true` | `amd64,arm64` | β€” (empty) | -| `branch=develop` | `develop` | payload `tag` | `true` | `amd64` | β€” (empty) | -| any other branch (PR) | `develop` | payload `tag` | `false` | `amd64` | `pr-N` / slug | -| `workflow_dispatch` | per ref | `latest` | `false`/`true`| `amd64` | per ref | +† `develop` is a **migration alias** kept so dispatches still in flight from the +pre-trunk monorepo land on staging instead of spinning up a stray preview Space. +Remove it once the trunk flip has been verified. -`env_name` selects the GitHub **Environment** (`main` vs `develop`), which is how -per-environment secrets/vars are scoped: `DOCKER_REPO`, `EXTRALIT_SERVER_IMAGE`, -`HF_SPACE_ID`, `HF_TOKEN`, `DOCKER_USERNAME`/`DOCKER_PASSWORD`. +`env_name` selects the GitHub **Environment** (`production` vs `staging`), which +is how per-environment secrets/vars are scoped: `DOCKER_REPO`, +`EXTRALIT_SERVER_IMAGE`, `HF_SPACE_ID`, `HF_TOKEN`, +`DOCKER_USERNAME`/`DOCKER_PASSWORD`. ### Job `build` Builds the self-contained Space image **on top of the server image**: @@ -210,8 +296,8 @@ The [`Dockerfile`](../../extralit-hf-space/Dockerfile) does `FROM ${EXTRALIT_SERVER_IMAGE}:${EXTRALIT_VERSION}` and layers on **Elasticsearch 8.17**, **Redis**, the **OCR/PDF-extraction** package, and a Procfile-based multi-process runtime (elastic + redis + RQ workers + FastAPI). Pushed to -`extralit/extralit-hf-space` (main) or `extralitdev/extralit-hf-space` (dev), -tagging `:latest` when `tag_latest=true`. +`extralit/extralit-hf-space` (production) or `extralitdev/extralit-hf-space` +(staging), tagging `:latest` when `tag_latest=true`. ### Job `deploy-space` β€” *non-PR builds only* (`pr_space_slug == ''`) Restarts the live Space so it pulls the freshly pushed image: @@ -221,66 +307,75 @@ curl -X POST "https://huggingface.co/api/spaces/${HF_SPACE_ID}/restart" \ -H "Authorization: Bearer $HF_TOKEN" ``` -`HF_SPACE_ID` is the environment-scoped Space (see Β§5): the **`main`** +`HF_SPACE_ID` is the environment-scoped Space (see Β§5): the **`production`** environment points at `extralit/public-demo` β€” the live public demo served at -**** β€” while the **`develop`** +**** β€” while the **`staging`** environment points at `extralit-dev/develop` (`extralit-dev-develop.hf.space`). ### Job `deploy-pr-space` β€” *PR builds only* (`pr_space_slug != ''`) -Runs under `environment: develop`. Creates an ephemeral preview Space per PR: +Runs under `environment: staging`. Creates an ephemeral preview Space per PR: 1. `duplicate_space("extralit-dev/develop" β†’ "extralit-dev/pr-N")` (cpu-basic) on first run, writing a README that enables `app_port: 6900`. 2. **Propagates config.** `duplicate_space` copies files but **not** secrets/variables, - so the job forwards the `develop` GitHub environment's `EXTRALIT_*` onto the Space β€” + so the job forwards the `staging` GitHub environment's `EXTRALIT_*` onto the Space β€” env **secrets** β†’ `add_space_secret`, env **variables** β†’ `add_space_variable`. It strictly filters to the `EXTRALIT_` prefix (via `toJSON(secrets)`/`toJSON(vars)`), so `HF_TOKEN`/`DOCKER_*`/the GitHub token are never pushed; only keys are logged. 3. Uploads a one-line `Dockerfile` (`FROM extralitdev/extralit-hf-space:pr-N`) so the preview Space tracks the PR's image. -> **OAuth is not wired for previews.** The custom HF OAuth app is pinned to the -> `extralit-dev-develop.hf.space` callback and can't serve ephemeral `pr-N` domains, so -> sign-in won't work on a preview. Do **not** put `EXTRALIT_ELASTICSEARCH`/ -> `EXTRALIT_REDIS_URL` in the `develop` env β€” the bundle runs its own ES/Redis at -> `localhost` and the generic filter would otherwise override them. +> **The Space `extralit-dev/develop` keeps its name.** It is a Hugging Face +> resource, not a git branch. The custom OAuth app is pinned to its +> `extralit-dev-develop.hf.space` callback, so renaming it breaks sign-in. This is +> the one place the word "develop" legitimately survives the trunk migration. + +> **OAuth is not wired for previews.** That same pinning means ephemeral `pr-N` +> domains can't serve the callback, so sign-in won't work on a preview. Do **not** +> put `EXTRALIT_ELASTICSEARCH`/`EXTRALIT_REDIS_URL` in the `staging` env β€” the +> bundle runs its own ES/Redis at `localhost` and the generic filter would +> otherwise override them. --- ## 5. Environment & secret reference -Resolved live from GitHub on 2026-06-10 via `gh api`. The `build-hf-space.yml` -jobs run with `environment: ${{ resolve-env.outputs.env_name }}`, so `vars.*` and -`secrets.*` resolve **per environment**. +The `build-hf-space.yml` jobs run with +`environment: ${{ resolve-env.outputs.env_name }}`, so `vars.*` and `secrets.*` +resolve **per environment**. ### `extralit/extralit-hf-space` β€” Environment **variables** (`vars.*`) -| Variable | `main` environment | `develop` environment | +| Variable | `production` environment | `staging` environment | | ---------------------- | ----------------------------- | ------------------------------- | | `DOCKER_REPO` | `extralit/extralit-hf-space` | `extralitdev/extralit-hf-space` | | `EXTRALIT_SERVER_IMAGE`| `extralit/extralit-server` | `extralitdev/extralit-server` | | `HF_SPACE_ID` | `extralit/public-demo` | `extralit-dev/develop` | +> These environments were previously named `main` and `develop`. They were +> renamed so the names describe the deploy target rather than a branch, and +> survive any future branch rename. + > The `HF_SPACE_ID` values map to Space URLs `-.hf.space`: > `extralit/public-demo` β†’ **extralit-public-demo.hf.space** (the public demo, -> deployed from `main`); `extralit-dev/develop` β†’ extralit-dev-develop.hf.space. -> There are **no** repo-level variables in this repo. +> deployed from `release`); `extralit-dev/develop` β†’ extralit-dev-develop.hf.space +> (deployed from `main`). There are **no** repo-level variables in this repo. -> **`EXTRALIT_*` on the `develop` env (consumed by `deploy-pr-space`).** Any -> `EXTRALIT_*` **variable** or **secret** added to the `develop` environment is forwarded +> **`EXTRALIT_*` on the `staging` env (consumed by `deploy-pr-space`).** Any +> `EXTRALIT_*` **variable** or **secret** added to the `staging` environment is forwarded > verbatim onto each `pr-N` Space (Β§4). Recommended: `EXTRALIT_DATABASE_URL`, > `EXTRALIT_S3_ACCESS_KEY`/`SECRET_KEY` and `EXTRALIT_AUTH_SECRET_KEY` as **secrets**; > `EXTRALIT_S3_ENDPOINT`/`REGION`/`SECURE`, `EXTRALIT_BASE_URL`, `EXTRALIT_CORS_ORIGINS` -> as **variables**. Set with `gh secret set --env develop` / -> `gh variable set --env develop --body `. +> as **variables**. Set with `gh secret set --env staging` / +> `gh variable set --env staging --body `. ### `extralit/extralit-hf-space` β€” **Secrets** (names only) -| Secret | Repo-level | `main` env | `develop` env | -| ----------------- | :--------: | :--------: | :-----------: | -| `HF_TOKEN` | βœ… (default) | βœ… (override) | βœ… (override) | -| `DOCKER_USERNAME` | β€” | βœ… | βœ… | -| `DOCKER_PASSWORD` | β€” | βœ… | βœ… | +| Secret | Repo-level | `production` env | `staging` env | +| ----------------- | :--------: | :--------------: | :-----------: | +| `HF_TOKEN` | βœ… (default) | βœ… (override) | βœ… (override) | +| `DOCKER_USERNAME` | β€” | βœ… | βœ… | +| `DOCKER_PASSWORD` | β€” | βœ… | βœ… | ### `extralit/extralit` (monorepo) @@ -294,6 +389,11 @@ jobs run with `environment: ${{ resolve-env.outputs.env_name }}`, so `vars.*` an defined at the repo or `HuggingFace`-environment level β€” they are inherited from **organization-level** secrets via `secrets: inherit` (reading them requires the `admin:org` scope). +- `release.yml` additionally needs a token that can push to `main`, `release`, and + tags β€” a GitHub App installation token or a PAT. `GITHUB_TOKEN` is **not** + sufficient: pushes made with it emit no downstream `push` events, so the tag + would never trigger the publish, docs, or GitHub-Release workflows. Under branch + protection this identity also needs a push bypass. ### Workflow-internal GHA env vars (computed at run time) @@ -304,18 +404,18 @@ the Actions logs when debugging a deploy. **Stage A β€” `extralit-server.build-docker-images.yml`** (`env:` set per `is_release`): -| Env var | Release (`main` / dispatch) | Dev (`develop` / PR) | +| Env var | Release (`release` branch) | Dev (`main` / PR) | | ------------------------ | -------------------------------- | --------------------------------- | | `IS_RELEASE` | `true` | `false` | | `PLATFORMS` | `linux/amd64,linux/arm64` | `linux/amd64` | | `IMAGE_TAG` | `v` | branch (cleaned) or `pr-N` | | `SERVER_DOCKER_IMAGE` | `extralit/extralit-server` | `extralitdev/extralit-server` | | `HF_SPACES_DOCKER_IMAGE` | `extralit/extralit-hf-space` | `extralitdev/extralit-hf-space` | -| `PUBLISH_LATEST` | `inputs.publish_latest` (main) | `true` (PR preview: `false`) | +| `PUBLISH_LATEST` | `inputs.publish_latest` | `true` (PR preview: `false`) | | `DOCKER_USERNAME/PASSWORD`| `AR_DOCKER_*` secrets | `AR_DOCKER_*_DEV` secrets | | `DISPATCH_BRANCH` | `github.ref_name` | `github.ref_name`, or `/merge` when `pr_number` set | -The optional `pr_number` input (set by `extralit-pr-preview.yml`'s manual path) +The optional `pr_number` input (set by `extralit-frontend.build-push-dev.yml`) forces `IMAGE_TAG=pr-N`, `PUBLISH_LATEST=false`, and `DISPATCH_BRANCH=/merge`. The cross-repo `client-payload` carries the handoff state: `tag` = `IMAGE_TAG`, `is_release` = `inputs.is_release`, `branch` = @@ -327,13 +427,13 @@ The cross-repo `client-payload` carries the handoff state: | --------------------------- | ----------------------------------------- | ----------------------------------------------- | | `EVENT_NAME` | `github.event_name` | `repository_dispatch` vs `workflow_dispatch` | | `PAYLOAD_TAG` | `client_payload.tag` | β†’ `image_tag` | -| `PAYLOAD_BRANCH` | `client_payload.branch` | branch routing | -| `PAYLOAD_IS_RELEASE` | `client_payload.is_release` | selects `main` env / multi-arch | -| `env_name` *(output)* | resolved from branch/payload | `environment:` β†’ which `vars.*`/`secrets.*` load | +| `PAYLOAD_BRANCH` | `client_payload.branch` | trunk vs preview routing | +| `PAYLOAD_IS_RELEASE` | `client_payload.is_release` | **the** production signal; selects `production` env / multi-arch | +| `env_name` *(output)* | resolved from payload | `environment:` β†’ which `vars.*`/`secrets.*` load | | `image_tag` *(output)* | payload tag, or `latest` (manual) | Docker tag built & deployed | -| `tag_latest` *(output)* | `true` on main/develop | also tag/push `:latest` | -| `platforms` *(output)* | `amd64` (dev) / `amd64,arm64` (release) | buildx target platforms | -| `pr_space_slug` *(output)* | `pr-N` / slug for non-main/develop | empty β†’ restart live Space; set β†’ PR preview | +| `tag_latest` *(output)* | `true` on release/trunk | also tag/push `:latest` | +| `platforms` *(output)* | `amd64` (staging) / `amd64,arm64` (release) | buildx target platforms | +| `pr_space_slug` *(output)* | `pr-N` / slug for preview refs | empty β†’ restart live Space; set β†’ PR preview | | `DOCKER_TAGS` | `${DOCKER_REPO}:${IMAGE_TAG}[,:latest]` | tags pushed by build job | | `EXTRALIT_SERVER_IMAGE` *(build-arg)* | `vars.EXTRALIT_SERVER_IMAGE` | base image the Space is built `FROM` | | `EXTRALIT_VERSION` *(build-arg)* | `image_tag` | base image tag (β†’ Dockerfile `ARG`) | @@ -348,60 +448,85 @@ The cross-repo `client-payload` carries the handoff state: ## 6. Gotchas & operational notes -- **Frontend changes don't auto-deploy the *public* demo.** A commit touching only - `extralit-frontend/**` runs `extralit-frontend.yml` (tests + artifact) but does - **not** trigger `extralit-server.yml` (path filter `extralit-server/**`), so it - doesn't redeploy `extralit/public-demo`. The shipped frontend is rebuilt from - source *inside* the server build. To roll a frontend-only change to the public - demo, merge to `main`/`develop` via the server flow. -- **PR previews cover frontend too.** Marking a PR (server **or** frontend) - **ready for review** triggers `extralit-pr-preview.yml` β†’ ephemeral - `extralit-dev/pr-N`. It does **not** rebuild on later pushes (by design); use - **Actions β†’ Deploy PR preview HF Space β†’ Run workflow** with the `pr_number` to - refresh. Previews are dev-org images and never touch `:latest` or production. -- **Two Docker orgs.** `extralit/*` = release (from `main`), `extralitdev/*` = - dev (from `develop`/PRs). The demo's `EXTRALIT_SERVER_IMAGE` env var picks - which base it builds on. +- **Nothing reaches production except a release.** Merging to `main` redeploys + only `extralit-dev/develop`. `extralit/public-demo` moves when β€” and only when β€” + `release.yml` is dispatched with `dry_run=false`. +- **Frontend changes don't auto-deploy the demo Spaces.** A commit touching only + `extralit-frontend/**` runs `extralit-frontend.yml` (tests + SPA artifact) but + does **not** trigger `extralit-server.yml` (path filter `extralit-server/**`). + The live UI goes to Vercel via its native Git integration; the SPA baked into + the server image only refreshes on the next server build, which picks up the + latest successful **`main`** frontend artifact β€” always `main`, release builds + included (see Β§3). +- **PR previews cover frontend too.** Marking a **non-fork** PR (server **or** + frontend) **ready for review** triggers `extralit-frontend.build-push-dev.yml` β†’ + ephemeral `extralit-dev/pr-N`. It does **not** rebuild on later pushes (by + design); use **Actions β†’ Deploy PR preview HF Space β†’ Run workflow** with the + `pr_number` to refresh. That manual path is also the only way to preview a + **fork** PR (Β§3a). Previews are dev-org images and never touch `:latest` or + production. +- **Two Docker orgs.** `extralit/*` = release (from `release`), `extralitdev/*` = + dev (from `main`/PRs). The Space's `EXTRALIT_SERVER_IMAGE` env var picks which + base it builds on. +- **A release fires `extralit-server.yml` three times.** The atomic push updates + `main`, `release` and the `vX.Y.Z` tag at the same SHA. The workflow's + `concurrency.group` is keyed on `github.ref` (not `github.sha`) precisely so + those runs don't cancel each other. They do different work: `release` builds and + ships the production image, the tag publishes to PyPI, and `main` rebuilds the + dev Space β€” one extra dev-Space rebuild per release is the accepted cost. - **Cross-repo token.** The handoff depends on `secrets.GH_ACTIONS_REPOSITORY_DISPATCH` (a PAT with dispatch rights on - `extralit/extralit-hf-space`). If the Space stops updating after server merges, + `extralit/extralit-hf-space`). If the Space stops updating after merges, check this token first. -- **Multi-arch only on release.** Dev/develop builds are `linux/amd64` only; - `arm64` is added only for `is_release` (main) builds. -- **Manual recovery.** You can rebuild/redeploy the Space directly from the - `extralit-hf-space` repo via **Actions β†’ Build & Deploy HF Space β†’ Run - workflow** (`workflow_dispatch`), bypassing the monorepo entirely. +- **Multi-arch only on release.** Trunk and PR builds are `linux/amd64` only; + `arm64` is added only for `is_release` builds. +- **Manual recovery.** You can rebuild/redeploy the **staging** Space directly + from the `extralit-hf-space` repo via **Actions β†’ Build & Deploy HF Space β†’ Run + workflow**, bypassing the monorepo. To redeploy **production**, re-run the + original dispatch-triggered run so its `client_payload` (and its `is_release` + flag) is replayed β€” a fresh manual dispatch always resolves to staging. --- ## 7. End-to-end summary -### Release β†’ public demo (`main`) - -1. PR merged to `main` touching `extralit-server/**`. -2. `extralit-server.yml` β†’ tests, builds frontend `dist`, bundles it, builds wheel. -3. `build_docker_images` (`is_release=true`) β†’ pushes - `extralit/extralit-server:v` (+`:latest`), multi-arch amd64+arm64. -4. `repository_dispatch(build-hf-space, {tag: v, is_release: true})` β†’ fires. -5. `build-hf-space.yml` β†’ `resolve-env` (env=`main`) β†’ builds - `extralit/extralit-hf-space:v` `FROM` the server image. -6. `deploy-space` β†’ `POST /spaces/extralit/public-demo/restart`. +### Trunk β†’ dev Space (`main`) + +1. PR squash-merged to `main` touching `extralit-server/**`. +2. `extralit-server.yml` β†’ tests, downloads the prebuilt SPA, bundles it, builds wheel. +3. `build_docker_images` (`is_release=false`) β†’ pushes + `extralitdev/extralit-server:main` (+`:latest`), amd64. +4. `repository_dispatch(build-hf-space, {tag: main, is_release: false, branch: main})`. +5. `build-hf-space.yml` β†’ `resolve-env` (env=`staging`) β†’ builds + `extralitdev/extralit-hf-space:main` `FROM` the server image. +6. `deploy-space` β†’ `POST /spaces/extralit-dev/develop/restart`. +7. Live at ****. **Production untouched.** + +### Release β†’ public demo (`release` + tag) + +1. `gh workflow run release.yml -f version=X.Y.Z -f dry_run=false`. +2. `release.yml` verifies authorization and green CI on `main`, stamps the version + via `scripts/bump_version.py set`, and pushes one commit to `main`, `release`, + and `vX.Y.Z` **atomically**. +3. The `release` push β†’ `extralit-server.yml` β†’ `build_docker_images` + (`is_release=true`) β†’ `extralit/extralit-server:vX.Y.Z` (+`:latest`), amd64+arm64. +4. `repository_dispatch(build-hf-space, {tag: vX.Y.Z, is_release: true})`. +5. `build-hf-space.yml` β†’ `resolve-env` (env=`production`) β†’ builds + `extralit/extralit-hf-space:vX.Y.Z` β†’ `deploy-space` restarts + `extralit/public-demo`. +6. In parallel, the **tag** push drives PyPI (`extralit`, `extralit-server`), + versioned docs (`mike deploy X.Y` + `stable`), and the GitHub Release. 7. Public demo live at ****. -### Staging (`develop`) - -Same flow, env=`develop`: `extralitdev/*` images tagged `develop` (+`:latest`, -amd64-only), restarting `extralit-dev/develop` β†’ extralit-dev-develop.hf.space. - ### PR preview β†’ ephemeral Space (`pr-N`) 1. A PR touching `extralit-server/**` or `extralit-frontend/**` is marked **ready for review** (or run manually with `pr_number=N`). -2. `extralit-pr-preview.yml` β†’ builds frontend `dist` + wheel (no pytest). +2. `extralit-frontend.build-push-dev.yml` β†’ downloads SPA + builds wheel (no pytest). 3. `build_docker_images` (`is_release=false`, `pr_number=N`) β†’ pushes `extralitdev/extralit-server:pr-N` (amd64, no `:latest`). 4. `repository_dispatch(build-hf-space, {tag: pr-N, is_release: false, branch: N/merge})`. -5. `build-hf-space.yml` β†’ `resolve-env` (env=`develop`, `pr_space_slug=pr-N`) β†’ builds +5. `build-hf-space.yml` β†’ `resolve-env` (env=`staging`, `pr_space_slug=pr-N`) β†’ builds `extralitdev/extralit-hf-space:pr-N`, then `deploy-pr-space` duplicates `extralit-dev/develop` β†’ **`extralit-dev/pr-N`** and points it at the image. 6. Preview live at **`https://extralit-dev-pr-N.hf.space`**. diff --git a/extralit-hf-space b/extralit-hf-space index fab9df541..35086a62d 160000 --- a/extralit-hf-space +++ b/extralit-hf-space @@ -1 +1 @@ -Subproject commit fab9df541d458f39128d610a90a05e9255c44e44 +Subproject commit 35086a62d14a62064d5891b03be39bdefcc6d6b1 diff --git a/extralit/docs/community/adding_language.md b/extralit/docs/community/adding_language.md index 8f5044f24..a8dc85821 100644 --- a/extralit/docs/community/adding_language.md +++ b/extralit/docs/community/adding_language.md @@ -32,7 +32,7 @@ export default { ### How to test it 1. Start a local instance of Extralit, easiest by just using the docker recipe [here](../getting_started/how-to-deploy-argilla-with-docker.md). It will give you a backend API for the frontend. -2. Compile a new version of the frontend. Check [this guide](https://github.com/extralit/extralit/tree/develop/extralit-frontend). This is basically: +2. Compile a new version of the frontend. Check [this guide](https://github.com/extralit/extralit/tree/main/extralit-frontend). This is basically: - `git clone https://github.com/extralit/extralit` - `cd extralit-frontend` - Install the dependencies: `npm i` diff --git a/extralit/docs/community/contributor.md b/extralit/docs/community/contributor.md index 87374b971..3351c6798 100644 --- a/extralit/docs/community/contributor.md +++ b/extralit/docs/community/contributor.md @@ -52,7 +52,7 @@ Below, you can see an example of the `Feature request` template. Once you choose After having reported the issue, you can start working on it. For that, you will need to create a fork of the project. To do that, click on the `Fork` button. -Now, fill in the information. Remember to uncheck the `Copy develop branch only` if you are going to work in or from another branch (for instance, to fix documentation the `main` branch is used). Then, click on `Create fork`. +Now, fill in the information. Leave `Copy the main branch only` checked β€” `main` is the only branch you need, whatever kind of change you're making. Then, click on `Create fork`. Now, you will be redirected to your fork. You can see that you are in your fork because the name of the repository will be your `username/extralit`, and it will indicate `forked from extralit/extralit`. @@ -71,13 +71,13 @@ cd extralit For each issue you're addressing, it's advisable to create a new branch. GitHub offers a straightforward method to streamline this process. -> ⚠️ Never work directly on the `main` or `develop` branch. Always create a new branch for your changes. +> ⚠️ Never work directly on the `main` branch. Always create a new branch for your changes. Navigate to your issue and on the right column, select `Create a branch`. -After the new window pops up, the branch will be named after the issue, include a prefix such as feature/, bug/, or docs/ to facilitate quick recognition of the issue type. In the `Repository destination`, pick your fork ( [your-github-username]/extralit), and then select `Change branch source` to specify the source branch for creating the new one. Complete the process by clicking `Create branch`. +After the new window pops up, the branch will be named after the issue, include a prefix such as `feat/`, `fix/`, or `docs/` to facilitate quick recognition of the issue type. In the `Repository destination`, pick your fork ( [your-github-username]/extralit), and then select `Change branch source` to specify the source branch for creating the new one. Complete the process by clicking `Create branch`. -> πŸ€” Remember that the `main` branch is only used to work with the documentation. For any other changes, use the `develop` branch. +> πŸ€” Extralit is trunk-based: `main` is the single source branch for everything β€” features, fixes, and documentation alike. Always set the branch source to `main`. Now, locally change to the new branch you just created. @@ -144,7 +144,7 @@ Come back to GitHub, navigate to the original repository where you created your First, click on `compare across forks` and select the right repositories and branches. -> In the base repository, keep in mind to select either `main` or `develop` based on the modifications made. In the head repository, indicate your forked repository and the branch corresponding to the issue. +> In the base repository, select `main` β€” it is the base for every kind of change. In the head repository, indicate your forked repository and the branch corresponding to the issue. Then, fill in the pull request template. You should add a prefix to the PR name as we did with the branch above. If you are working on a new feature, you can name your PR as `feat: TITLE`. If your PR consists of a solution for a bug, you can name your PR as `bug: TITLE` And, if your work is for improving the documentation, you can name your PR as `docs: TITLE`. @@ -168,7 +168,7 @@ Congratulations πŸŽ‰πŸŽŠ We thank you 🀩 Once your PR is merged, your contributions will be publicly visible on the [Extralit GitHub](https://github.com/extralit/extralit#contributors). -Additionally, we will include your changes in the next release based on our [development branch](https://github.com/extralit/extralit/tree/develop). +Additionally, your changes ship in the next release cut from our [trunk](https://github.com/extralit/extralit/tree/main). ## Additional resources diff --git a/extralit/docs/community/developer.md b/extralit/docs/community/developer.md index c701a1b91..437e993f9 100644 --- a/extralit/docs/community/developer.md +++ b/extralit/docs/community/developer.md @@ -44,11 +44,11 @@ Once you have your environment set up, you can return to this guide to learn mor The Extralit repository has a monorepo structure, which means that all the components are located in the same repository: [`extralit/extralit`](https://github.com/extralit/extralit). This repo is divided into the following folders: -- [`extralit/src/extralit/`](https://github.com/extralit/extralit/tree/develop/extralit): The Extralit SDK -- [`extralit/docs/`](https://github.com/extralit/extralit/tree/develop/extralit/docs): The documentation project -- [`extralit-server/src/extralit_server/`](https://github.com/extralit/extralit/tree/develop/extralit-server): The FastAPI server project for annotation -- [`extralit-frontend/`](https://github.com/extralit/extralit/tree/develop/extralit-frontend): The Vue.js annotation UI project -- [`examples`](https://github.com/extralit/extralit/tree/develop/examples): Example resources for deployments, scripts and notebooks +- [`extralit/src/extralit/`](https://github.com/extralit/extralit/tree/main/extralit): The Extralit SDK +- [`extralit/docs/`](https://github.com/extralit/extralit/tree/main/extralit/docs): The documentation project +- [`extralit-server/src/extralit_server/`](https://github.com/extralit/extralit/tree/main/extralit-server): The FastAPI server project for annotation +- [`extralit-frontend/`](https://github.com/extralit/extralit/tree/main/extralit-frontend): The Vue.js annotation UI project +- [`examples`](https://github.com/extralit/extralit/tree/main/examples): Example resources for deployments, scripts and notebooks !!! note "How to contribute?" Before starting to develop, we recommend reading our [contribution guide](contributor.md) to understand the contribution process and the guidelines to follow. Once you have [cloned the Extralit repository](contributor.md#fork-the-extralit-repository) and [checked out to the correct branch](contributor.md#create-a-new-branch), you can start setting up your development environment. @@ -144,8 +144,8 @@ uv run pytest tests/integration --disable-warnings --cov=extralit Documentation is essential to provide users with a comprehensive guide about Extralit. -!!! note "From `main` or `develop`?" - If you are updating, improving, or fixing the current documentation without a code change, work on the `main` branch. For new features or bug fixes that require documentation, use the `develop` branch. +!!! note "Which branch?" + `main` β€” always. Extralit is trunk-based, so documentation-only changes and code changes both branch from `main` and merge back into it. Docs published from `main` appear at `docs.extralit.ai/latest`; a `docs/**` branch additionally gets its own hidden preview build. To contribute to the documentation and generate it locally, ensure you installed the development dependencies as shown in the ["Set up the Python environment"](#set-up-the-python-environment) section, and run the following command to create the development server with `mkdocs`: @@ -281,7 +281,7 @@ app.add_typer(mycommand.app, name="mycommand") - Create commands that fit into existing workflows - Follow consistent naming and structure patterns -- Provide clear help text for all commands and options, e.g. use the [`print_rich_table`](https://github.com/extralit/extralit/blob/develop/extralit/src/extralit/cli/rich.py#L115) function to print tables in a rich format +- Provide clear help text for all commands and options, e.g. use the [`print_rich_table`](https://github.com/extralit/extralit/blob/main/extralit/src/extralit/cli/rich.py#L115) function to print tables in a rich format - Use sensible defaults to minimize required input - Follow the Unix philosophy: commands should do one thing well diff --git a/extralit/docs/community/index.md b/extralit/docs/community/index.md index 44a6ca9dc..7bee861d9 100644 --- a/extralit/docs/community/index.md +++ b/extralit/docs/community/index.md @@ -49,7 +49,7 @@ We are an open-source community-driven project focused on building a platform th The changelog is where you can find the latest updates and changes to the Extralit project. - [:octicons-arrow-right-24: Changelog β†—](https://github.com/extralit/extralit/blob/develop/extralit/CHANGELOG.md) + [:octicons-arrow-right-24: Changelog β†—](https://github.com/extralit/extralit/blob/main/extralit/CHANGELOG.md) - __Roadmap__ diff --git a/extralit/docs/community/release_guide.md b/extralit/docs/community/release_guide.md index d66560df5..5730183c4 100644 --- a/extralit/docs/community/release_guide.md +++ b/extralit/docs/community/release_guide.md @@ -6,62 +6,78 @@ hide: # Extralit Release Guide -This guide provides a simplified, step-by-step process for creating a new release of Extralit. Follow these steps to ensure a smooth and consistent release process. +Releasing Extralit is one workflow dispatch. You do not create branches, edit version files, push tags, or draft release notes by hand β€” `release.yml` does all of it, and refuses to run if the state isn't right. -**Tips:** -- Always update the version in `src/extralit/_version.py` before tagging. -- Use clear, descriptive release notes. -- Coordinate with other maintainers if needed. +## Cut a release -## 1. Prepare the Release Branch +```sh +# 1. Rehearse. Validates everything and prints a plan, but pushes nothing. +gh workflow run release.yml -f version=0.7.0 -- Ensure all features and fixes for the release are merged into `develop`. -- Create a release branch from `develop`: - ```sh - git checkout develop - git pull origin develop - git checkout -b releases/vX.Y.Z - git push origin releases/vX.Y.Z - ``` +# 2. Read the plan in the run summary, then cut it for real. +gh workflow run release.yml -f version=0.7.0 -f dry_run=false +``` -## 2. Open Pull Requests +`dry_run` defaults to **true**, so the first command is the safe default and a bare `gh workflow run release.yml -f version=…` can never publish anything. Watch either run with `gh run watch`. -- Open a PR from `releases/vX.Y.Z` into `develop` (if any last-minute fixes are needed), merge it. -- Open a PR from `develop` into `main`. -- Use "Squash and merge" for a clean history if desired. +The plan table in the run summary tells you the version, the tag, the exact commit on `main` being released, whether the release is already converged, and whether the run is in `DRY RUN` or `EXECUTE` mode. Read it before step 2. -## 3. Merge and Tag the Release +You can also run both from the **Actions β†’ Release** page if you prefer the UI. -- After merging into `main`, checkout `main` locally and pull the latest changes: - ```sh - git checkout main - git pull origin main - ``` -- Tag the release: - ```sh - git tag vX.Y.Z - git push origin vX.Y.Z - ``` +## What happens automatically -## 4. Create the GitHub Release +Once `dry_run=false` succeeds, one commit stamping the version lands on `main`, `release`, and the tag `v0.7.0` β€” all three at the same SHA, pushed atomically. That single push then drives everything else in parallel: -- Go to the [GitHub Releases page](https://github.com/extralit/extralit/releases). -- Click "Draft a new release". -- Set the tag to `vX.Y.Z`. -- Add release notes (see previous releases for examples). -- Publish the release. +| Trigger | Result | +| --- | --- | +| push to `release` | multi-arch `extralit/extralit-server:v0.7.0` + `:latest`, then the HF Space rebuild that restarts **`extralit/public-demo`** | +| push to tag `v0.7.0` | `extralit` and `extralit-server` published to PyPI | +| push to tag `v0.7.0` | versioned docs at `docs.extralit.ai/v0.7/`, with `stable` re-pointed at it | +| push to tag `v0.7.0` | a GitHub Release with notes generated from merged PR titles | -## 5. Verify the Release +The GitHub Release step waits until PyPI actually serves both packages before publishing, so a release is never announced before it's installable. -- Monitor GitHub Actions to ensure the release workflow completes successfully. -- Check [PyPI](https://pypi.org/project/extralit/) to confirm the new version is published. -- Test the CLI: - ```sh - pip install --upgrade extralit - extralit --help - ``` +## Verify -## 6. Announce the Release +```sh +git ls-remote origin refs/heads/main refs/heads/release refs/tags/v0.7.0 # all three at one SHA +pip index versions extralit +curl -s https://extralit-public-demo.hf.space/api/v1/status | jq .version # 0.7.0 +gh release view v0.7.0 +``` -- Share the release notes with the community (Slack, GitHub Discussions, etc.). +Docs land at [docs.extralit.ai/v0.7/](https://docs.extralit.ai/v0.7/), and `stable` should redirect there. +## If something goes wrong + +The workflow reports exactly which precondition failed. + +**"Tag v0.7.0 already exists at a different commit."** A previous attempt got partway. If that tag is genuinely wrong, delete it and re-run: + +```sh +git push origin :refs/tags/v0.7.0 && git tag -d v0.7.0 +``` + +If the tag is correct and already points at a commit stamped `0.7.0`, the workflow instead reports **`Converged already: true`** and does nothing. Re-running a completed release is always safe β€” that's the property that makes recovery possible. + +**"CI is not green on `main`."** The workflow refuses to release an untested commit and tells you whether checks are failing, still running, or absent entirely. Wait for them, or override deliberately: + +```sh +gh workflow run release.yml -f version=0.7.0 -f dry_run=false -f skip_ci_check=true +``` + +**"Not authorized."** Cutting a release requires `admin` or `maintain` on the repository. + +## Roll back + +Production follows the `release` branch, so rolling back means moving it and restarting the Space β€” no revert commit, no new version: + +```sh +git push --force-with-lease origin v0.6.1^{commit}:refs/heads/release +``` + +That rebuilds and redeploys `extralit/public-demo` from the previous release. PyPI releases cannot be un-published β€” if a bad version reached PyPI, yank it there and cut a fixed patch release instead. + +## Announce + +Share the generated release notes with the community (Slack, GitHub Discussions). diff --git a/extralit/docs/scripts/gen_changelog.py b/extralit/docs/scripts/gen_changelog.py index 49ee808d7..efc7fde84 100644 --- a/extralit/docs/scripts/gen_changelog.py +++ b/extralit/docs/scripts/gen_changelog.py @@ -6,7 +6,7 @@ REPOSITORY = "Extralit/extralit" CHANGELOG_PATH = "extralit/CHANGELOG.md" -RETRIEVED_BRANCH = "develop" +RETRIEVED_BRANCH = "main" DATA_PATH = "community/changelog.md" diff --git a/scripts/bump_version.py b/scripts/bump_version.py new file mode 100644 index 000000000..cfbdd6f62 --- /dev/null +++ b/scripts/bump_version.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Single owner of the project version, which lives in three hand-synced files. + + python scripts/bump_version.py check # print the version, fail if they disagree + python scripts/bump_version.py check --expect 0.7.0 # post-condition assertion + python scripts/bump_version.py set --version 0.7.0 # rewrite all three + +`check` prints the bare version to stdout and everything else to stderr, so CI can do +`V=$(python scripts/bump_version.py check)`. Called by the release workflow and by the +build workflows that previously each grepped `_version.py` by hand. + +Stdlib only, and paths resolve relative to this file rather than the CWD β€” workflow steps +run it from `extralit-server/`, `extralit/`, and the repo root. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# X.Y.Z only. Deliberately matches the validation in .github/workflows/release.yml so the +# two can't disagree about what a releasable version looks like. +SEMVER = re.compile(r"^\d+\.\d+\.\d+$") + +# `^__version__ = "..."` anchored at line start so a version string inside a docstring or +# comment can't be picked up. +PY_VERSION = re.compile(r'^(__version__\s*=\s*")([^"]*)(")', re.MULTILINE) + +# The first `"version": "..."` in package.json. Anchored to line start; the result is +# re-parsed as JSON afterwards to prove we hit the top-level key and not a nested one. +JSON_VERSION = re.compile(r'^(\s*"version"\s*:\s*")([^"]*)(")', re.MULTILINE) + + +@dataclass(frozen=True) +class VersionFile: + path: Path + pattern: re.Pattern + is_json: bool = False + + @property + def rel(self) -> str: + return str(self.path.relative_to(REPO_ROOT)) + + def read(self) -> str: + if not self.path.exists(): + die(f"{self.rel}: file not found") + text = self.path.read_text() + if self.is_json: + # Authoritative for reads: no regex ambiguity about which key we got. + try: + version = json.loads(text).get("version") + except json.JSONDecodeError as exc: + die(f"{self.rel}: invalid JSON ({exc})") + if not isinstance(version, str): + die(f'{self.rel}: no top-level string "version" key') + return version + match = self.pattern.search(text) + if not match: + die(f'{self.rel}: no `__version__ = "..."` assignment found') + return match.group(2) + + def write(self, version: str) -> bool: + """Rewrite in place. Returns True if the file changed.""" + # `cmd_set` writes before it reads anything, so without this a missing file would make a + # raw FileNotFoundError traceback the first output of a release run. + if not self.path.exists(): + die(f"{self.rel}: file not found") + text = self.path.read_text() + new_text, count = self.pattern.subn( + lambda m: f"{m.group(1)}{version}{m.group(3)}", text, count=1 + ) + if count != 1: + die(f"{self.rel}: version pattern did not match; refusing to guess") + if self.is_json: + # Prove the substitution landed on the top-level key rather than a nested one. + try: + parsed = json.loads(new_text) + except json.JSONDecodeError as exc: + die(f"{self.rel}: rewrite produced invalid JSON ({exc})") + if parsed.get("version") != version: + die( + f'{self.rel}: rewrite hit a nested "version" key, not the top-level one ' + f"(top-level is still {parsed.get('version')!r})" + ) + if new_text == text: + return False + self.path.write_text(new_text) + return True + + +FILES = ( + VersionFile( + REPO_ROOT / "extralit" / "src" / "extralit" / "_version.py", PY_VERSION + ), + VersionFile( + REPO_ROOT / "extralit-server" / "src" / "extralit_server" / "_version.py", + PY_VERSION, + ), + VersionFile( + REPO_ROOT / "extralit-frontend" / "package.json", JSON_VERSION, is_json=True + ), +) + + +def die(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + raise SystemExit(1) + + +def current_versions() -> dict[str, str]: + return {f.rel: f.read() for f in FILES} + + +def cmd_check(args: argparse.Namespace) -> int: + versions = current_versions() + distinct = set(versions.values()) + + if len(distinct) != 1: + print("error: version files disagree:", file=sys.stderr) + for rel, version in versions.items(): + print(f" {version} {rel}", file=sys.stderr) + print("run `bump_version.py set --version X.Y.Z` to resync", file=sys.stderr) + return 1 + + version = distinct.pop() + + if args.expect is not None and version != args.expect: + print(f"error: expected {args.expect}, found {version}", file=sys.stderr) + return 1 + + for rel in versions: + print(f" {version} {rel}", file=sys.stderr) + print(version) + return 0 + + +def cmd_set(args: argparse.Namespace) -> int: + version = args.version + if not SEMVER.match(version): + die(f"{version!r} is not X.Y.Z") + + changed = [] + for f in FILES: + if f.write(version): + changed.append(f.rel) + + if changed: + for rel in changed: + print(f" updated {rel}", file=sys.stderr) + else: + print(f"already at {version}; nothing to do", file=sys.stderr) + + # Post-condition: never claim success on a partial rewrite. + versions = current_versions() + if set(versions.values()) != {version}: + die(f"post-condition failed, files are now inconsistent: {versions}") + + print(version) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + + p_check = sub.add_parser( + "check", help="print the version; fail if the files disagree" + ) + p_check.add_argument( + "--expect", metavar="X.Y.Z", help="also assert the version equals this" + ) + p_check.set_defaults(func=cmd_check) + + p_set = sub.add_parser("set", help="rewrite the version in every file") + p_set.add_argument("--version", required=True, metavar="X.Y.Z") + p_set.set_defaults(func=cmd_set) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main())