From 90d51dff83b84390e074ff4db8570f60e55792ed Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 13:09:04 -0500 Subject: [PATCH 01/37] ci: Add PR previews via Codespaces Recovers the codespaces preview work from the pr-codespace branch and updates it to the current repo: mise-pinned node 24 / pnpm 11, the current realm layout (base/catalog/skills/openrouter), corrected prerender-manager (4222) and worker-manager (4210) ports, and the RESOLVED_OPENROUTER_REALM_URL host build var. Postgres now uses the repo's own infra:ensure-pg (boxel-pg) via docker-in-docker rather than a separate compose service. The realm server is launched by hand to run plain HTTP (TLS terminates at the GitHub port-forwarding edge), bypassing the mandatory local dev cert, and with REALM_SERVER_SKIP_BOOT_INDEX so readiness doesn't block on a from-scratch index that would need a prerender host the Codespace doesn't run. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/Dockerfile | 15 ++ .devcontainer/devcontainer.json | 48 +++++++ .devcontainer/setup.sh | 35 +++++ .devcontainer/start-services.sh | 175 +++++++++++++++++++++++ .github/workflows/codespaces-preview.yml | 112 +++++++++++++++ packages/host/config/codespaces.env | 11 ++ 6 files changed, 396 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/setup.sh create mode 100755 .devcontainer/start-services.sh create mode 100644 .github/workflows/codespaces-preview.yml create mode 100644 packages/host/config/codespaces.env diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..9feb4cf2a2 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,15 @@ +# Node major here only needs to be close enough to bootstrap mise; the exact +# toolchain (node + pnpm) is pinned in /.mise.toml and installed by setup.sh. +FROM mcr.microsoft.com/devcontainers/javascript-node:24 + +# mise manages the Node and pnpm versions pinned in the repo's .mise.toml. +# `mise install` is run by .devcontainer/setup.sh once the repo is checked out. +RUN curl https://mise.run | sh && \ + echo 'eval "$(~/.local/bin/mise activate bash)"' >> /home/node/.bashrc + +# System dependencies for node-gyp. Postgres, Matrix/Synapse and SMTP run as +# Docker containers (via the docker-in-docker feature) started by the repo's +# own mise tasks — no system packages needed for them here. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..3f0d25cb70 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,48 @@ +{ + "name": "Boxel PR Review", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "workspaceFolder": "/workspaces/boxel", + + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "dockerDashComposeVersion": "v2" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.11" + }, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + + "forwardPorts": [4201, 4206, 8008], + "portsAttributes": { + "4201": { "label": "Realm Server", "onAutoForward": "silent" }, + "4206": { "label": "Icons Server", "onAutoForward": "silent" }, + "8008": { "label": "Matrix/Synapse", "onAutoForward": "silent" } + }, + + "postCreateCommand": "bash .devcontainer/setup.sh", + "postStartCommand": "bash .devcontainer/start-services.sh", + + "customizations": { + "vscode": { + "extensions": [ + "cardstack.boxel-tools", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "typed-ember.glint-vscode" + ], + "settings": { + "terminal.integrated.defaultProfile.linux": "bash" + } + } + }, + + "hostRequirements": { + "cpus": 4, + "memory": "8gb", + "storage": "64gb" + } +} diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100755 index 0000000000..31ac83e02e --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# One-time setup after the container is created. +# Runs during Codespace build (or prebuild) — keep it idempotent. +# The host app is NOT built here; it's deployed via GitHub Actions +# (.github/workflows/codespaces-preview.yml) pointed back at this Codespace. +set -euo pipefail + +cd /workspaces/boxel + +# mise installs the exact Node + pnpm versions pinned in .mise.toml. `mise +# trust` is required because the repo's .mise.toml has not been trusted in a +# fresh container. Activate mise for this shell so the pinned tools are on PATH. +echo "==> Installing pinned toolchain via mise..." +~/.local/bin/mise trust +~/.local/bin/mise install +eval "$(~/.local/bin/mise activate bash)" + +echo "==> Installing dependencies..." +mise exec -- pnpm install --frozen-lockfile + +# Source-realm content lives in separate repos that are cloned on first setup. +# In a Codespace the GitHub token has access to these, so the https clone in +# each :setup script succeeds. These are also re-run (idempotently) when the +# realm server starts, but doing them here moves the clone cost into setup. +echo "==> Setting up skills realm..." +mise exec -- pnpm --dir=packages/skills-realm skills:setup + +echo "==> Setting up catalog realm..." +mise exec -- pnpm --dir=packages/catalog catalog:setup +mise exec -- pnpm --dir=packages/catalog catalog:update + +# Database schema is created on demand: infra:ensure-pg starts the boxel-pg +# container and creates the databases, and the realm server runs with +# --migrateDB to apply migrations. Both happen in start-services.sh. +echo "==> Setup complete. Backend services will start automatically." diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh new file mode 100755 index 0000000000..7f02f27885 --- /dev/null +++ b/.devcontainer/start-services.sh @@ -0,0 +1,175 @@ +#!/bin/bash +# Start backend services for PR review in Codespaces. +# +# The reviewer-facing host app is NOT built here — a GitHub Actions workflow +# (.github/workflows/codespaces-preview.yml) builds it and deploys it to S3 +# with URLs pointing back at this Codespace's forwarded ports. +# +# The services run plain HTTP locally; GitHub's port forwarding terminates +# TLS at the edge (https://-.app.github.dev). This deliberately +# bypasses the repo's standard local-dev HTTPS path (mkcert + the mandatory +# `infra:ensure-dev-cert`), which is why the realm server is launched by hand +# below rather than via `mise run dev` / `start:development`. +# +# Known limitation: card *prerendering* needs a host app for the prerenderer +# to render against, which is not run in the Codespace. The realm server is +# therefore started in mount-and-serve mode (REALM_SERVER_SKIP_BOOT_INDEX), +# so it comes up immediately and serves modules/source; prerendered card +# rendering and search are degraded until a host is wired up for the +# prerenderer (follow-up). +set -euo pipefail + +cd /workspaces/boxel + +# mise provides the pinned node/pnpm/ts-node toolchain. +eval "$(~/.local/bin/mise activate bash)" + +CODESPACE_NAME="${CODESPACE_NAME:?CODESPACE_NAME must be set}" +FWD_DOMAIN="${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" + +# Public Codespace URLs for the forwarded ports — TLS terminated at the +# GitHub edge. These become the realms' public identities (toUrl) and are +# passed to the host build so the S3-hosted preview can reach this backend. +export REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" +export MATRIX_PUBLIC_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" +export ICONS_PUBLIC_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" + +# Internal service-to-service wiring is plain HTTP on the standard ports. +# Exported so the repo's mise service tasks (which source +# mise-tasks/lib/env-vars.sh) target HTTP localhost rather than the default +# https://localhost:4201 — there is no dev cert here, so HTTPS would fail. +export REALM_BASE_URL="http://localhost:4201" +export MATRIX_URL="http://localhost:8008" +export MATRIX_URL_VAL="http://localhost:8008" +export ICONS_URL="http://localhost:4206" +export PGPORT=5435 +export PGDATABASE=boxel + +# Mount-and-serve: skip the from-scratch boot index (see "Known limitation" +# above). Without this, a brand-new realm's readiness check blocks on a full +# index that needs the prerenderer + a host, which the Codespace lacks. +export REALM_SERVER_SKIP_BOOT_INDEX=true + +# ── Make forwarded ports public so the S3 preview can reach them ── +echo "==> Making forwarded ports public..." +gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true + +# ── Postgres (boxel-pg container) + databases ── +echo "==> Ensuring Postgres is running..." +mise run infra:ensure-pg + +# ── Matrix/Synapse ── +echo "==> Starting Matrix/Synapse..." +(cd packages/matrix && MATRIX_URL=http://localhost:8008 pnpm assert-synapse-running) & +SYNAPSE_PID=$! + +# ── SMTP (MailHog) ── +echo "==> Starting SMTP server..." +(cd packages/matrix && pnpm assert-smtp-running) & + +# ── Icons / prerender / worker (best-effort; see "Known limitation") ── +echo "==> Starting icons server..." +pnpm --dir=packages/realm-server run start:icons & + +echo "==> Starting prerender services..." +pnpm --dir=packages/realm-server run start:prerender-manager-dev & +pnpm --dir=packages/realm-server run start:prerender-dev & + +echo "==> Starting worker..." +pnpm --dir=packages/realm-server run start:worker-development & + +# Wait for Synapse before starting the realm server (it registers the +# realm_server Matrix user during boot). +wait $SYNAPSE_PID || true + +# Synapse's registration shared secret is needed to register the +# realm_server Matrix user. Read it from the running Synapse config, the +# same way mise-tasks/services/realm-server does. +if [ -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]; then + MATRIX_REGISTRATION_SHARED_SECRET=$(pnpm --dir=packages/realm-server exec ts-node --transpileOnly scripts/matrix-registration-secret.ts) + export MATRIX_REGISTRATION_SHARED_SECRET +fi + +# ── Realm server ── +# Launched by hand (not via mise) to run plain HTTP and skip the dev-cert +# requirement. toUrls are the public Codespace URLs so the S3 host resolves +# realms back to this backend. Realm layout matches mise-tasks/services/ +# realm-server: base, catalog, skills, openrouter (experiments / homepage / +# submission / software-factory are skipped to keep the preview lean). +echo "==> Starting realm server..." +SKIP_EXPERIMENTS=true \ +SKIP_BOXEL_HOMEPAGE=true \ +SKIP_SUBMISSION=true \ +SKIP_SOFTWARE_FACTORY=true \ +NODE_ENV=development \ +NODE_NO_WARNINGS=1 \ +PGPORT=5435 \ +PGDATABASE=boxel \ +LOG_LEVELS='*=info' \ +REALM_SERVER_SECRET_SEED="mum's the word" \ +REALM_SECRET_SEED="shhh! it's a secret" \ +MATRIX_URL=http://localhost:8008 \ +REALM_SERVER_MATRIX_USERNAME=realm_server \ +ENABLE_FILE_WATCHER=true \ +REALM_SERVER_SKIP_BOOT_INDEX=true \ + pnpm --dir=packages/realm-server exec ts-node \ + --transpileOnly main \ + --port=4201 \ + --matrixURL=http://localhost:8008 \ + --realmsRootPath=./realms/codespaces \ + --prerendererUrl=http://localhost:4222 \ + --migrateDB \ + --workerManagerPort=4210 \ + \ + --path='../base' \ + --username='base_realm' \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl="${REALM_SERVER_URL}/base/" \ + \ + --path='../catalog/contents' \ + --username='catalog_realm' \ + --fromUrl='@cardstack/catalog/' \ + --toUrl="${REALM_SERVER_URL}/catalog/" \ + \ + --path='../skills-realm/contents' \ + --username='skills_realm' \ + --fromUrl='@cardstack/skills/' \ + --toUrl="${REALM_SERVER_URL}/skills/" \ + \ + --path='../openrouter-realm' \ + --username='openrouter_realm' \ + --fromUrl='@cardstack/openrouter/' \ + --toUrl="${REALM_SERVER_URL}/openrouter/" & +REALM_PID=$! + +# ── Wait for realm server readiness ── +echo "==> Waiting for realm server to be ready..." +timeout 300 bash -c \ + 'until curl -sf "http://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ + || echo "Warning: realm server readiness check timed out after 5 minutes" + +# ── Trigger host preview build via GitHub Actions ── +echo "==> Triggering host preview build pointed at this Codespace..." +BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" +gh workflow run codespaces-preview.yml \ + --ref "$BRANCH_NAME" \ + -f codespace_name="$CODESPACE_NAME" \ + -f realm_server_url="$REALM_SERVER_URL" \ + -f matrix_url="$MATRIX_PUBLIC_URL" \ + -f icons_url="$ICONS_PUBLIC_URL" \ + || echo "Warning: could not trigger preview build. Run manually with: gh workflow run codespaces-preview.yml" + +echo "" +echo "============================================" +echo " Backend services running!" +echo "" +echo " Realm server: ${REALM_SERVER_URL}" +echo " Matrix: ${MATRIX_PUBLIC_URL}" +echo " Icons: ${ICONS_PUBLIC_URL}" +echo "" +echo " Host preview build triggered — check the" +echo " PR for a preview link once it completes." +echo "============================================" + +# Keep the realm server (and this script) in the foreground. +wait "$REALM_PID" diff --git a/.github/workflows/codespaces-preview.yml b/.github/workflows/codespaces-preview.yml new file mode 100644 index 0000000000..1ca6bfc6a2 --- /dev/null +++ b/.github/workflows/codespaces-preview.yml @@ -0,0 +1,112 @@ +name: Codespaces Preview + +on: + workflow_dispatch: + inputs: + codespace_name: + description: "Codespace name (used to derive forwarded port URLs)" + required: true + type: string + realm_server_url: + description: "Codespace realm server URL (e.g. https://-4201.app.github.dev)" + required: true + type: string + matrix_url: + description: "Codespace Matrix URL (e.g. https://-8008.app.github.dev)" + required: true + type: string + icons_url: + description: "Codespace icons URL (e.g. https://-4206.app.github.dev)" + required: true + type: string + +permissions: + contents: read + issues: write + checks: write + pull-requests: write + id-token: write + statuses: write + +jobs: + deploy-codespaces-preview: + name: Build and deploy host preview for Codespaces + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: ./.github/actions/init + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 + with: + role-to-assume: arn:aws:iam::680542703984:role/boxel-host + aws-region: us-east-1 + + - name: Load staging app environment variables as a base + shell: bash + run: cat packages/host/config/staging.env >> $GITHUB_ENV + + - name: Override backend URLs to point at the Codespace + shell: bash + run: | + # These override the staging.env values loaded above (last write + # wins in $GITHUB_ENV) so the host build talks to this Codespace's + # forwarded backend services instead of staging. + echo "REALM_SERVER_DOMAIN=${{ inputs.realm_server_url }}/" >> $GITHUB_ENV + echo "RESOLVED_BASE_REALM_URL=${{ inputs.realm_server_url }}/base/" >> $GITHUB_ENV + echo "RESOLVED_CATALOG_REALM_URL=${{ inputs.realm_server_url }}/catalog/" >> $GITHUB_ENV + echo "RESOLVED_SKILLS_REALM_URL=${{ inputs.realm_server_url }}/skills/" >> $GITHUB_ENV + echo "RESOLVED_OPENROUTER_REALM_URL=${{ inputs.realm_server_url }}/openrouter/" >> $GITHUB_ENV + echo "MATRIX_URL=${{ inputs.matrix_url }}" >> $GITHUB_ENV + echo "MATRIX_SERVER_NAME=localhost" >> $GITHUB_ENV + echo "ICONS_URL=${{ inputs.icons_url }}" >> $GITHUB_ENV + + - name: Set PR branch name for S3 prefix + shell: bash + run: | + RAW_BRANCH="${GITHUB_REF_NAME}" + echo "PR_BRANCH_NAME=$(echo "${RAW_BRANCH}" | tr _ - | tr '[:upper:]' '[:lower:]' | sed -e 's/-$//' | sed -e 's/[^a-z0-9\-]//g' | cut -c1-60)" >> $GITHUB_ENV + + - name: Build and deploy preview + shell: bash + env: + S3_PREVIEW_BUCKET_NAME: boxel-host-preview.stack.cards + AWS_S3_BUCKET: boxel-host-preview.stack.cards + AWS_REGION: us-east-1 + AWS_CLOUDFRONT_DISTRIBUTION: EU4RGLH4EOCHJ + run: pnpm deploy:boxel-host:preview-staging + + - name: Store preview URL + shell: bash + run: echo "PREVIEW_HOST=https://${PR_BRANCH_NAME}.boxel-host-preview.stack.cards/" >> $GITHUB_ENV + + - name: Post status check with preview link + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + HEAD_SHA: ${{ github.sha }} + run: | + curl \ + -X POST \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$REPOSITORY/statuses/$HEAD_SHA" \ + -d '{"context":"Preview boxel-host codespaces","description":"Host preview connected to Codespace backend","target_url":"'"$PREVIEW_HOST"'","state":"success"}' + + - name: Find associated PR and comment + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + PR_NUMBER=$(gh pr list --head "${GITHUB_REF_NAME}" --json number --jq '.[0].number' 2>/dev/null || true) + if [ -n "$PR_NUMBER" ]; then + gh pr comment "$PR_NUMBER" --body "### Codespaces Preview + + - [Host preview (connected to Codespace)]($PREVIEW_HOST) + - Realm server: ${{ inputs.realm_server_url }} + - Matrix: ${{ inputs.matrix_url }} + + > This preview build is connected to the Codespace \`${{ inputs.codespace_name }}\`. The preview will stop working when the Codespace is stopped." + fi diff --git a/packages/host/config/codespaces.env b/packages/host/config/codespaces.env new file mode 100644 index 0000000000..2594e9f631 --- /dev/null +++ b/packages/host/config/codespaces.env @@ -0,0 +1,11 @@ +# These values are overridden at build time by the codespaces-preview workflow. +# They serve as documentation of the host build env vars a Codespace preview +# needs — see packages/host/config/environment.js for how they are consumed. +REALM_SERVER_DOMAIN=http://localhost:4201/ +RESOLVED_BASE_REALM_URL=http://localhost:4201/base/ +RESOLVED_CATALOG_REALM_URL=http://localhost:4201/catalog/ +RESOLVED_SKILLS_REALM_URL=http://localhost:4201/skills/ +RESOLVED_OPENROUTER_REALM_URL=http://localhost:4201/openrouter/ +MATRIX_URL=http://localhost:8008 +MATRIX_SERVER_NAME=localhost +ICONS_URL=http://localhost:4206 From 614fa3ad61cb3f184cad30ce0f0febfb2816d53f Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 13:42:50 -0500 Subject: [PATCH 02/37] ci: trigger Codespaces preview on push instead of dispatch workflow_dispatch can only be triggered for workflows on the default branch, so the preview couldn't be exercised before merge. Switch to a push trigger on the preview branch: start-services.sh records the Codespace name + forwarding domain in .devcontainer/codespace-target.env and commits/pushes it, and the workflow reads that file to derive the forwarded backend URLs. The Codespace's startup push triggers the first build and later code pushes rebuild against the same backend. A guard skips the build when no target file is present, and workflow_dispatch remains as a manual fallback (now needing only the Codespace name). Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 35 +++++++--- .github/workflows/codespaces-preview.yml | 86 ++++++++++++++++++------ 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 7f02f27885..3488d4149d 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -148,16 +148,33 @@ timeout 300 bash -c \ 'until curl -sf "http://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ || echo "Warning: realm server readiness check timed out after 5 minutes" -# ── Trigger host preview build via GitHub Actions ── -echo "==> Triggering host preview build pointed at this Codespace..." +# ── Record this Codespace as the preview target ── +# The codespaces-preview workflow triggers on pushes to this branch and reads +# the committed target file to point the host build at this Codespace's +# forwarded backend. Committing + pushing it triggers the first build; every +# later code push then rebuilds against the same backend. +echo "==> Recording Codespace target for the preview workflow..." +TARGET_FILE=".devcontainer/codespace-target.env" +cat > "$TARGET_FILE" </dev/null + git push origin "HEAD:${BRANCH_NAME}" \ + || echo "Warning: could not push Codespace target; the preview build was not triggered." +fi echo "" echo "============================================" diff --git a/.github/workflows/codespaces-preview.yml b/.github/workflows/codespaces-preview.yml index 1ca6bfc6a2..9aa21537c5 100644 --- a/.github/workflows/codespaces-preview.yml +++ b/.github/workflows/codespaces-preview.yml @@ -1,23 +1,26 @@ name: Codespaces Preview on: + # Pushes to the preview branch rebuild the host preview against whatever + # Codespace is currently recorded in .devcontainer/codespace-target.env. + # That file is written + committed by .devcontainer/start-services.sh when + # a Codespace boots, so the Codespace's own startup push triggers the first + # build and every subsequent code push rebuilds against the same backend. + # Add other branches here to enable Codespace previews on them. + push: + branches: + - codespaces-10255 + # Manual fallback — supply the Codespace name directly. workflow_dispatch: inputs: codespace_name: description: "Codespace name (used to derive forwarded port URLs)" required: true type: string - realm_server_url: - description: "Codespace realm server URL (e.g. https://-4201.app.github.dev)" - required: true - type: string - matrix_url: - description: "Codespace Matrix URL (e.g. https://-8008.app.github.dev)" - required: true - type: string - icons_url: - description: "Codespace icons URL (e.g. https://-4206.app.github.dev)" - required: true + forwarding_domain: + description: "Codespaces port-forwarding domain" + required: false + default: app.github.dev type: string permissions: @@ -35,40 +38,78 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Resolve Codespace target + shell: bash + run: | + # Determine the Codespace name from the dispatch input or the + # committed target file, then derive the forwarded backend URLs. + # If neither is available (e.g. a code push before any Codespace + # has booted), skip the rest of the job cleanly. + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + NAME="${{ inputs.codespace_name }}" + DOMAIN="${{ inputs.forwarding_domain }}" + elif [ -f .devcontainer/codespace-target.env ]; then + # shellcheck disable=SC1091 + . .devcontainer/codespace-target.env + NAME="${CODESPACE_NAME:-}" + DOMAIN="${CODESPACE_FORWARDING_DOMAIN:-}" + else + NAME="" + fi + DOMAIN="${DOMAIN:-app.github.dev}" + if [ -z "$NAME" ]; then + echo "No Codespace target recorded; skipping preview build." + echo "SHOULD_RUN=false" >> "$GITHUB_ENV" + exit 0 + fi + { + echo "SHOULD_RUN=true" + echo "CODESPACE_NAME=$NAME" + echo "REALM_SERVER_URL=https://$NAME-4201.$DOMAIN" + echo "MATRIX_PUBLIC_URL=https://$NAME-8008.$DOMAIN" + echo "ICONS_PUBLIC_URL=https://$NAME-4206.$DOMAIN" + } >> "$GITHUB_ENV" + - uses: ./.github/actions/init + if: env.SHOULD_RUN == 'true' - name: Configure AWS credentials + if: env.SHOULD_RUN == 'true' uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: role-to-assume: arn:aws:iam::680542703984:role/boxel-host aws-region: us-east-1 - name: Load staging app environment variables as a base + if: env.SHOULD_RUN == 'true' shell: bash run: cat packages/host/config/staging.env >> $GITHUB_ENV - name: Override backend URLs to point at the Codespace + if: env.SHOULD_RUN == 'true' shell: bash run: | # These override the staging.env values loaded above (last write # wins in $GITHUB_ENV) so the host build talks to this Codespace's # forwarded backend services instead of staging. - echo "REALM_SERVER_DOMAIN=${{ inputs.realm_server_url }}/" >> $GITHUB_ENV - echo "RESOLVED_BASE_REALM_URL=${{ inputs.realm_server_url }}/base/" >> $GITHUB_ENV - echo "RESOLVED_CATALOG_REALM_URL=${{ inputs.realm_server_url }}/catalog/" >> $GITHUB_ENV - echo "RESOLVED_SKILLS_REALM_URL=${{ inputs.realm_server_url }}/skills/" >> $GITHUB_ENV - echo "RESOLVED_OPENROUTER_REALM_URL=${{ inputs.realm_server_url }}/openrouter/" >> $GITHUB_ENV - echo "MATRIX_URL=${{ inputs.matrix_url }}" >> $GITHUB_ENV + echo "REALM_SERVER_DOMAIN=${REALM_SERVER_URL}/" >> $GITHUB_ENV + echo "RESOLVED_BASE_REALM_URL=${REALM_SERVER_URL}/base/" >> $GITHUB_ENV + echo "RESOLVED_CATALOG_REALM_URL=${REALM_SERVER_URL}/catalog/" >> $GITHUB_ENV + echo "RESOLVED_SKILLS_REALM_URL=${REALM_SERVER_URL}/skills/" >> $GITHUB_ENV + echo "RESOLVED_OPENROUTER_REALM_URL=${REALM_SERVER_URL}/openrouter/" >> $GITHUB_ENV + echo "MATRIX_URL=${MATRIX_PUBLIC_URL}" >> $GITHUB_ENV echo "MATRIX_SERVER_NAME=localhost" >> $GITHUB_ENV - echo "ICONS_URL=${{ inputs.icons_url }}" >> $GITHUB_ENV + echo "ICONS_URL=${ICONS_PUBLIC_URL}" >> $GITHUB_ENV - name: Set PR branch name for S3 prefix + if: env.SHOULD_RUN == 'true' shell: bash run: | RAW_BRANCH="${GITHUB_REF_NAME}" echo "PR_BRANCH_NAME=$(echo "${RAW_BRANCH}" | tr _ - | tr '[:upper:]' '[:lower:]' | sed -e 's/-$//' | sed -e 's/[^a-z0-9\-]//g' | cut -c1-60)" >> $GITHUB_ENV - name: Build and deploy preview + if: env.SHOULD_RUN == 'true' shell: bash env: S3_PREVIEW_BUCKET_NAME: boxel-host-preview.stack.cards @@ -78,10 +119,12 @@ jobs: run: pnpm deploy:boxel-host:preview-staging - name: Store preview URL + if: env.SHOULD_RUN == 'true' shell: bash run: echo "PREVIEW_HOST=https://${PR_BRANCH_NAME}.boxel-host-preview.stack.cards/" >> $GITHUB_ENV - name: Post status check with preview link + if: env.SHOULD_RUN == 'true' shell: bash env: GITHUB_TOKEN: ${{ github.token }} @@ -96,6 +139,7 @@ jobs: -d '{"context":"Preview boxel-host codespaces","description":"Host preview connected to Codespace backend","target_url":"'"$PREVIEW_HOST"'","state":"success"}' - name: Find associated PR and comment + if: env.SHOULD_RUN == 'true' shell: bash env: GH_TOKEN: ${{ github.token }} @@ -105,8 +149,8 @@ jobs: gh pr comment "$PR_NUMBER" --body "### Codespaces Preview - [Host preview (connected to Codespace)]($PREVIEW_HOST) - - Realm server: ${{ inputs.realm_server_url }} - - Matrix: ${{ inputs.matrix_url }} + - Realm server: ${REALM_SERVER_URL} + - Matrix: ${MATRIX_PUBLIC_URL} - > This preview build is connected to the Codespace \`${{ inputs.codespace_name }}\`. The preview will stop working when the Codespace is stopped." + > This preview build is connected to the Codespace \`${CODESPACE_NAME}\`. The preview will stop working when the Codespace is stopped." fi From 511c0f08fee2a1ca55c419df4ec1eeecc62f9e4b Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 14:35:41 -0500 Subject: [PATCH 03/37] ci: fix Codespace build on Debian trixie base image The floating javascript-node:24 tag now resolves to Debian trixie, where the docker-in-docker feature's default Moby packages are unavailable and container creation fails before any setup runs. Pin the base image to bookworm and set the feature's moby option to false (installs Docker CE from Docker's official repo) per the feature's own error guidance. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/Dockerfile | 5 ++++- .devcontainer/devcontainer.json | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9feb4cf2a2..682a93fbe3 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,9 @@ # Node major here only needs to be close enough to bootstrap mise; the exact # toolchain (node + pnpm) is pinned in /.mise.toml and installed by setup.sh. -FROM mcr.microsoft.com/devcontainers/javascript-node:24 +# Pinned to bookworm: the floating `:24` tag now resolves to Debian trixie, +# which several devcontainer features (notably docker-in-docker's Moby +# packages) don't yet support. +FROM mcr.microsoft.com/devcontainers/javascript-node:24-bookworm # mise manages the Node and pnpm versions pinned in the repo's .mise.toml. # `mise install` is run by .devcontainer/setup.sh once the repo is checked out. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3f0d25cb70..7249271c94 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,7 +8,8 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": { - "dockerDashComposeVersion": "v2" + "dockerDashComposeVersion": "v2", + "moby": false }, "ghcr.io/devcontainers/features/python:1": { "version": "3.11" From 8e99cbfc47988003d93eec6a732fab253bcf1ca8 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 15:06:48 -0500 Subject: [PATCH 04/37] ci: run Codespace start-services detached so postStart returns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit start-services.sh blocks on the realm server to keep its child services alive, but as a postStartCommand that never returns it wedges the Codespaces agent — SSH never comes up and the Codespace sits unusable in "Available". Launch it detached via nohup from postStartCommand (logging to /tmp/start-services.log) so the lifecycle hook returns immediately while the services keep running. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/devcontainer.json | 2 +- .devcontainer/start-services.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7249271c94..600b7b3838 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,7 +25,7 @@ }, "postCreateCommand": "bash .devcontainer/setup.sh", - "postStartCommand": "bash .devcontainer/start-services.sh", + "postStartCommand": "nohup bash .devcontainer/start-services.sh > /tmp/start-services.log 2>&1 &", "customizations": { "vscode": { diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 3488d4149d..88a303afab 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -188,5 +188,9 @@ echo " Host preview build triggered — check the" echo " PR for a preview link once it completes." echo "============================================" -# Keep the realm server (and this script) in the foreground. +# This script is launched detached (nohup ... &) from postStartCommand so the +# Codespace lifecycle hook returns immediately — a blocking postStartCommand +# wedges the Codespaces agent and SSH never comes up. Blocking on the realm +# server here keeps this script alive as the parent of all the backgrounded +# services for the life of the Codespace. Output goes to /tmp/start-services.log. wait "$REALM_PID" From d5f19d7d60e130fe103e2b888266213600d2e5be Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 15:24:41 -0500 Subject: [PATCH 05/37] ci: install mise globally and grant Codespace access to content repos postCreate runs as the 'node' user but the Dockerfile's mise install ran as root, leaving the binary under /root where node can't see it (exit 127). Install mise to /usr/local/bin (on PATH for every user) and call it unqualified from the setup/start scripts. Also grant the Codespace read access to cardstack/boxel-catalog and cardstack/boxel-skills so setup.sh's catalog:setup / skills:setup clones authenticate; a Codespace token only reaches its own repo by default. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/Dockerfile | 8 ++++++-- .devcontainer/devcontainer.json | 6 ++++++ .devcontainer/setup.sh | 6 +++--- .devcontainer/start-services.sh | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 682a93fbe3..9fe202262d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,8 +7,12 @@ FROM mcr.microsoft.com/devcontainers/javascript-node:24-bookworm # mise manages the Node and pnpm versions pinned in the repo's .mise.toml. # `mise install` is run by .devcontainer/setup.sh once the repo is checked out. -RUN curl https://mise.run | sh && \ - echo 'eval "$(~/.local/bin/mise activate bash)"' >> /home/node/.bashrc +# Install to a global path: this RUN executes as root, so the default +# ~/.local/bin/mise would land under /root and be invisible to the 'node' user +# that Codespaces runs lifecycle commands as. A global binary is on PATH for +# every user; per-user mise data still lives under each user's home. +RUN curl https://mise.run | MISE_INSTALL_PATH=/usr/local/bin/mise sh && \ + echo 'eval "$(mise activate bash)"' >> /etc/bash.bashrc # System dependencies for node-gyp. Postgres, Matrix/Synapse and SMTP run as # Docker containers (via the docker-in-docker feature) started by the repo's diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 600b7b3838..ed78fc0847 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -28,6 +28,12 @@ "postStartCommand": "nohup bash .devcontainer/start-services.sh > /tmp/start-services.log 2>&1 &", "customizations": { + "codespaces": { + "repositories": { + "cardstack/boxel-catalog": { "permissions": { "contents": "read" } }, + "cardstack/boxel-skills": { "permissions": { "contents": "read" } } + } + }, "vscode": { "extensions": [ "cardstack.boxel-tools", diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 31ac83e02e..1a18774341 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -11,9 +11,9 @@ cd /workspaces/boxel # trust` is required because the repo's .mise.toml has not been trusted in a # fresh container. Activate mise for this shell so the pinned tools are on PATH. echo "==> Installing pinned toolchain via mise..." -~/.local/bin/mise trust -~/.local/bin/mise install -eval "$(~/.local/bin/mise activate bash)" +mise trust +mise install +eval "$(mise activate bash)" echo "==> Installing dependencies..." mise exec -- pnpm install --frozen-lockfile diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 88a303afab..bd5dab4ba6 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -22,7 +22,7 @@ set -euo pipefail cd /workspaces/boxel # mise provides the pinned node/pnpm/ts-node toolchain. -eval "$(~/.local/bin/mise activate bash)" +eval "$(mise activate bash)" CODESPACE_NAME="${CODESPACE_NAME:?CODESPACE_NAME must be set}" FWD_DOMAIN="${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" From 4420036601d4f5d3a66b0508142086ca00f9bb4a Mon Sep 17 00:00:00 2001 From: Codespace Preview Date: Wed, 17 Jun 2026 20:47:04 +0000 Subject: [PATCH 06/37] ci: record Codespace preview target --- .devcontainer/codespace-target.env | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .devcontainer/codespace-target.env diff --git a/.devcontainer/codespace-target.env b/.devcontainer/codespace-target.env new file mode 100644 index 0000000000..50fda4fcbb --- /dev/null +++ b/.devcontainer/codespace-target.env @@ -0,0 +1,5 @@ +# Written by .devcontainer/start-services.sh when a Codespace boots. +# The codespaces-preview workflow reads this to point the host build at this +# Codespace's forwarded backend services. Safe to delete; do not merge to main. +CODESPACE_NAME=vigilant-space-engine-74gxpg9qfp766 +CODESPACE_FORWARDING_DOMAIN=app.github.dev From f81ed70a63fe72258d687b3d31d458cd9337bab9 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 15:34:12 -0500 Subject: [PATCH 07/37] ci: clone Codespace content repos over HTTPS, not SSH The catalog/skills :setup scripts attempt an SSH clone first, which hangs on an interactive host-key prompt in the non-interactive postCreate context (a Codespace has an HTTPS token credential helper but no SSH key). Rewrite git@github.com: URLs to https:// before the clones so they authenticate with the token instead of blocking. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/setup.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 1a18774341..349743f494 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -19,9 +19,15 @@ echo "==> Installing dependencies..." mise exec -- pnpm install --frozen-lockfile # Source-realm content lives in separate repos that are cloned on first setup. -# In a Codespace the GitHub token has access to these, so the https clone in -# each :setup script succeeds. These are also re-run (idempotently) when the -# realm server starts, but doing them here moves the clone cost into setup. +# The catalog/skills :setup scripts try an SSH clone (git@github.com:) first, +# which blocks on an interactive host-key prompt in this non-interactive +# context. A Codespace has an HTTPS token credential helper but no SSH key, +# so rewrite SSH GitHub URLs to HTTPS — the clones then authenticate with the +# token (the repos are granted in devcontainer.json customizations.codespaces). +# These are also re-run idempotently when the realm server starts; doing them +# here moves the clone cost into setup. +git config --global url."https://github.com/".insteadOf "git@github.com:" + echo "==> Setting up skills realm..." mise exec -- pnpm --dir=packages/skills-realm skills:setup From bf0fea13020d58683559942a404ef1f6563bd479 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 15:47:39 -0500 Subject: [PATCH 08/37] ci: set GRAFANA_SECRET for the Codespace realm server The hand-rolled realm-server launch omitted GRAFANA_SECRET, which main.ts treats as required (process.exit(-1)), so the server died at startup and readiness never passed. Set it (and LOW_CREDIT_THRESHOLD) to match the canonical mise-tasks/services/realm-server invocation. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index bd5dab4ba6..891837cdc0 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -108,6 +108,8 @@ PGDATABASE=boxel \ LOG_LEVELS='*=info' \ REALM_SERVER_SECRET_SEED="mum's the word" \ REALM_SECRET_SEED="shhh! it's a secret" \ +GRAFANA_SECRET="shhh! it's a secret" \ +LOW_CREDIT_THRESHOLD="${LOW_CREDIT_THRESHOLD:-2000}" \ MATRIX_URL=http://localhost:8008 \ REALM_SERVER_MATRIX_USERNAME=realm_server \ ENABLE_FILE_WATCHER=true \ From 745145e2f0cdc4d9428280b8ca737b7190404167 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 16:03:47 -0500 Subject: [PATCH 09/37] ci: fix docker-in-docker install (drop moby:false, skip buildx) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit moby:false routed the feature through Docker CE, whose buildx v0.35.0 artifact download 404s and fails the build. It was only added to dodge the Debian trixie issue, which the bookworm pin already fixes — so the default Moby path works now. Drop moby:false and disable the buildx install (the Codespace only runs containers, never builds images). Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ed78fc0847..1a1e606dfb 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,7 +9,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": { "dockerDashComposeVersion": "v2", - "moby": false + "installDockerBuildx": false }, "ghcr.io/devcontainers/features/python:1": { "version": "3.11" From e2a6cf43f1aefe8f8c061316639bef498ba25ed7 Mon Sep 17 00:00:00 2001 From: Codespace Preview Date: Wed, 17 Jun 2026 21:32:37 +0000 Subject: [PATCH 10/37] ci: record Codespace preview target --- .devcontainer/codespace-target.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/codespace-target.env b/.devcontainer/codespace-target.env index 50fda4fcbb..673184e5bd 100644 --- a/.devcontainer/codespace-target.env +++ b/.devcontainer/codespace-target.env @@ -1,5 +1,5 @@ # Written by .devcontainer/start-services.sh when a Codespace boots. # The codespaces-preview workflow reads this to point the host build at this # Codespace's forwarded backend services. Safe to delete; do not merge to main. -CODESPACE_NAME=vigilant-space-engine-74gxpg9qfp766 +CODESPACE_NAME=humble-spoon-v7x5q4w5cq44 CODESPACE_FORWARDING_DOMAIN=app.github.dev From 3b9e4942aba89c06473ee92b0dc0e5909faf00f1 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 16:35:33 -0500 Subject: [PATCH 11/37] ci: run a local host app so the Codespace realm server boots and cards render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The realm server hard-requires a reachable host: main.ts fetches HOST_URL at startup and process.exit(-2)s if it can't, and the prerenderer renders cards against it. The Codespace ran no host, so the realm server failed and prerendering couldn't launch Chrome. Run the host (vite) on http://localhost:4200 with the public Codespace realm URLs, install the Chromium shared libraries puppeteer's bundled Chrome needs, set HOST_URL so the realm distURL and prerender target the local http host, and start the host first — waiting for it before the prerender and realm server. Bump the machine to 16 GB for the added load. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/Dockerfile | 12 ++++-- .devcontainer/devcontainer.json | 2 +- .devcontainer/start-services.sh | 69 ++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9fe202262d..5ebe22e384 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,9 +14,15 @@ FROM mcr.microsoft.com/devcontainers/javascript-node:24-bookworm RUN curl https://mise.run | MISE_INSTALL_PATH=/usr/local/bin/mise sh && \ echo 'eval "$(mise activate bash)"' >> /etc/bash.bashrc -# System dependencies for node-gyp. Postgres, Matrix/Synapse and SMTP run as -# Docker containers (via the docker-in-docker feature) started by the repo's -# own mise tasks — no system packages needed for them here. +# build-essential for node-gyp; the rest are the shared libraries Chromium +# needs so puppeteer's bundled Chrome can launch for card prerendering +# (the prerender service fails with "libatk-1.0.so.0: cannot open shared +# object file" without them). Postgres, Matrix/Synapse and SMTP run as +# Docker containers via the docker-in-docker feature, so they need nothing here. RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ + libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \ + libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \ + libgbm1 libpango-1.0-0 libcairo2 libasound2 libatspi2.0-0 \ + libgtk-3-0 libx11-xcb1 libxcb1 libxss1 fonts-liberation \ && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1a1e606dfb..14ee9cd9b1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -49,7 +49,7 @@ "hostRequirements": { "cpus": 4, - "memory": "8gb", + "memory": "16gb", "storage": "64gb" } } diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 891837cdc0..5779062e41 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -11,12 +11,14 @@ # `infra:ensure-dev-cert`), which is why the realm server is launched by hand # below rather than via `mise run dev` / `start:development`. # -# Known limitation: card *prerendering* needs a host app for the prerenderer -# to render against, which is not run in the Codespace. The realm server is -# therefore started in mount-and-serve mode (REALM_SERVER_SKIP_BOOT_INDEX), -# so it comes up immediately and serves modules/source; prerendered card -# rendering and search are degraded until a host is wired up for the -# prerenderer (follow-up). +# A host app (vite) IS run locally here, on http://localhost:4200, because +# the realm server hard-requires a reachable host: main.ts fetches HOST_URL +# at startup and process.exit(-2)s if it can't (and the prerenderer renders +# cards against it). This local host is distinct from the reviewer-facing S3 +# build; it exists so the realm server boots and cards can prerender. The +# realm still starts in mount-and-serve mode (REALM_SERVER_SKIP_BOOT_INDEX) +# so readiness doesn't block on a full from-scratch index; cards prerender +# on demand instead. set -euo pipefail cd /workspaces/boxel @@ -45,15 +47,43 @@ export ICONS_URL="http://localhost:4206" export PGPORT=5435 export PGDATABASE=boxel -# Mount-and-serve: skip the from-scratch boot index (see "Known limitation" -# above). Without this, a brand-new realm's readiness check blocks on a full -# index that needs the prerenderer + a host, which the Codespace lacks. +# Mount-and-serve: skip the from-scratch boot index so readiness doesn't +# block on a full index of every bootstrap realm; cards prerender on demand. export REALM_SERVER_SKIP_BOOT_INDEX=true +# The realm server's distURL and the prerenderer both target the local host +# over plain HTTP (no dev cert here). main.ts defaults distURL to +# https://localhost:4200, which would fail against the http vite host, so +# pin it explicitly. The mise prerender task honours this via ${HOST_URL:-…}. +export HOST_URL="http://localhost:4200" +# Give the prerender's puppeteer standby probe headroom for vite's cold start. +export PRERENDER_STANDBY_TIMEOUT_MS="${PRERENDER_STANDBY_TIMEOUT_MS:-120000}" + # ── Make forwarded ports public so the S3 preview can reach them ── echo "==> Making forwarded ports public..." gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true +# ── Host app (vite) ── +# Started first because it's the slowest to warm and both the realm server +# (distURL smoke test) and the prerenderer depend on it. It runs with the +# PUBLIC realm/matrix/icons URLs so prerendered output references the +# Codespace's public hostnames (which the S3 reviewer host resolves), even +# though it is itself served on plain-HTTP localhost:4200. +echo "==> Starting host app (vite) on http://localhost:4200..." +( + cd packages/host + REALM_SERVER_DOMAIN="${REALM_SERVER_URL}/" \ + RESOLVED_BASE_REALM_URL="${REALM_SERVER_URL}/base/" \ + RESOLVED_CATALOG_REALM_URL="${REALM_SERVER_URL}/catalog/" \ + RESOLVED_SKILLS_REALM_URL="${REALM_SERVER_URL}/skills/" \ + RESOLVED_OPENROUTER_REALM_URL="${REALM_SERVER_URL}/openrouter/" \ + MATRIX_URL="${MATRIX_PUBLIC_URL}" \ + MATRIX_SERVER_NAME=localhost \ + ICONS_URL="${ICONS_PUBLIC_URL}" \ + exec node scripts/vite-serve.js +) > /tmp/host-vite.log 2>&1 & +HOST_PID=$! + # ── Postgres (boxel-pg container) + databases ── echo "==> Ensuring Postgres is running..." mise run infra:ensure-pg @@ -67,17 +97,30 @@ SYNAPSE_PID=$! echo "==> Starting SMTP server..." (cd packages/matrix && pnpm assert-smtp-running) & -# ── Icons / prerender / worker (best-effort; see "Known limitation") ── +# ── Icons / worker ── echo "==> Starting icons server..." pnpm --dir=packages/realm-server run start:icons & +echo "==> Starting worker..." +pnpm --dir=packages/realm-server run start:worker-development & + +# Wait for the host app before starting the prerenderer (its puppeteer +# standby probe loads the host) and the realm server (its distURL smoke +# test fetches the host). vite cold-starts the full host dep graph, so +# allow generous time; a missing host is fatal to the realm server. +echo "==> Waiting for host app at http://localhost:4200 (vite cold start)..." +if ! timeout 420 bash -c 'until curl -sf http://localhost:4200 >/dev/null 2>&1; do + kill -0 '"$HOST_PID"' 2>/dev/null || { echo "host vite process died; see /tmp/host-vite.log"; exit 1; } + sleep 3 + done'; then + echo "Warning: host app not reachable; realm smoke test and prerender will likely fail. See /tmp/host-vite.log" +fi + +# ── Prerender (needs the host to be up) ── echo "==> Starting prerender services..." pnpm --dir=packages/realm-server run start:prerender-manager-dev & pnpm --dir=packages/realm-server run start:prerender-dev & -echo "==> Starting worker..." -pnpm --dir=packages/realm-server run start:worker-development & - # Wait for Synapse before starting the realm server (it registers the # realm_server Matrix user during boot). wait $SYNAPSE_PID || true From fb2bb2dc0bccfcf2cd25af81015185936947a494 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 16:37:32 -0500 Subject: [PATCH 12/37] ci: detach Codespace start-services with setsid so it isn't reaped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain backgrounded postStartCommand (nohup &) was reaped after the hook returned — the services never ran and the log stayed empty. Run it under setsid in its own session (stdin from /dev/null) so a process-group cleanup of the lifecycle hook doesn't take it down. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 14ee9cd9b1..a0cb6e2364 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,7 +25,7 @@ }, "postCreateCommand": "bash .devcontainer/setup.sh", - "postStartCommand": "nohup bash .devcontainer/start-services.sh > /tmp/start-services.log 2>&1 &", + "postStartCommand": "setsid bash .devcontainer/start-services.sh > /tmp/start-services.log 2>&1 < /dev/null &", "customizations": { "codespaces": { From 85b29f304cfbd926e219b5305754ece2c204fa74 Mon Sep 17 00:00:00 2001 From: Codespace Preview Date: Wed, 17 Jun 2026 22:06:52 +0000 Subject: [PATCH 13/37] ci: record Codespace preview target --- .devcontainer/codespace-target.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/codespace-target.env b/.devcontainer/codespace-target.env index 673184e5bd..7a77345f2e 100644 --- a/.devcontainer/codespace-target.env +++ b/.devcontainer/codespace-target.env @@ -1,5 +1,5 @@ # Written by .devcontainer/start-services.sh when a Codespace boots. # The codespaces-preview workflow reads this to point the host build at this # Codespace's forwarded backend services. Safe to delete; do not merge to main. -CODESPACE_NAME=humble-spoon-v7x5q4w5cq44 +CODESPACE_NAME=didactic-space-funicular-gwgp7vq43w9jx CODESPACE_FORWARDING_DOMAIN=app.github.dev From a42c7f3ee6842f4672e8786686b243d442f2717b Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 17:26:49 -0500 Subject: [PATCH 14/37] ci: serve the Codespace host and realm over HTTPS to match the prerender The prerender pipeline is HTTPS-only: it probes https://localhost:4200 and passes --ignore-certificate-errors. Running the vite host over plain HTTP produced ERR_SSL_PROTOCOL_ERROR, and the http overrides for HOST_URL / REALM_BASE_URL never reached the mise services anyway (mise re-sources env-vars.sh without the ambient exports, so they kept the https defaults). Generate a self-signed cert at the path env-vars.sh probes, so vite, the realm server and the prerender all speak HTTPS consistently; trust it in Node via NODE_EXTRA_CA_CERTS. Drop the http overrides and probe readiness with https + -k. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/setup.sh | 19 ++++++++ .devcontainer/start-services.sh | 82 ++++++++++++++++++--------------- 2 files changed, 64 insertions(+), 37 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 349743f494..151a0d0c8a 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -35,6 +35,25 @@ echo "==> Setting up catalog realm..." mise exec -- pnpm --dir=packages/catalog catalog:setup mise exec -- pnpm --dir=packages/catalog catalog:update +# The realm server, vite host and prerenderer all speak HTTPS in this repo +# (browsers only do HTTP/2 over TLS, which the prerender pipeline assumes). +# Generate a self-signed cert at the path env-vars.sh probes +# (~/.local/share/boxel/dev-certs) so it sets REALM_SERVER_TLS_CERT_FILE and +# HOST_URL=https for every mise service, and vite serves HTTPS too. The +# prerender's puppeteer ignores cert errors; Node trusts it via +# NODE_EXTRA_CA_CERTS (set in start-services.sh). mkcert isn't used — nothing +# here needs the cert in a browser/system trust store. +echo "==> Generating self-signed dev TLS cert..." +CERT_DIR="$HOME/.local/share/boxel/dev-certs" +mkdir -p "$CERT_DIR" +if [ ! -f "$CERT_DIR/localhost.pem" ]; then + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$CERT_DIR/localhost-key.pem" \ + -out "$CERT_DIR/localhost.pem" \ + -days 365 -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" +fi + # Database schema is created on demand: infra:ensure-pg starts the boxel-pg # container and creates the databases, and the realm server runs with # --migrateDB to apply migrations. Both happen in start-services.sh. diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 5779062e41..ca36e65e6a 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -5,20 +5,21 @@ # (.github/workflows/codespaces-preview.yml) builds it and deploys it to S3 # with URLs pointing back at this Codespace's forwarded ports. # -# The services run plain HTTP locally; GitHub's port forwarding terminates -# TLS at the edge (https://-.app.github.dev). This deliberately -# bypasses the repo's standard local-dev HTTPS path (mkcert + the mandatory -# `infra:ensure-dev-cert`), which is why the realm server is launched by hand -# below rather than via `mise run dev` / `start:development`. +# Services run HTTPS locally (realm server :4201, vite host :4200) using the +# self-signed cert generated in setup.sh — the repo's prerender pipeline is +# HTTPS-only, so a plain-HTTP host yields ERR_SSL_PROTOCOL_ERROR. GitHub's +# port forwarding tunnels those https services out at the edge +# (https://-.app.github.dev). The realm server is launched by +# hand (not `mise run dev`) only so its realm toUrls can be the public +# Codespace URLs; it still picks up the HTTPS cert via env-vars.sh. # -# A host app (vite) IS run locally here, on http://localhost:4200, because -# the realm server hard-requires a reachable host: main.ts fetches HOST_URL -# at startup and process.exit(-2)s if it can't (and the prerenderer renders -# cards against it). This local host is distinct from the reviewer-facing S3 -# build; it exists so the realm server boots and cards can prerender. The -# realm still starts in mount-and-serve mode (REALM_SERVER_SKIP_BOOT_INDEX) -# so readiness doesn't block on a full from-scratch index; cards prerender -# on demand instead. +# A host app (vite) IS run locally because the realm server hard-requires a +# reachable host: main.ts fetches HOST_URL at startup and process.exit(-2)s +# if it can't (and the prerenderer renders cards against it). This local host +# is distinct from the reviewer-facing S3 build; it exists so the realm server +# boots and cards can prerender. The realm starts in mount-and-serve mode +# (REALM_SERVER_SKIP_BOOT_INDEX) so readiness doesn't block on a full +# from-scratch index; cards prerender on demand instead. set -euo pipefail cd /workspaces/boxel @@ -36,26 +37,27 @@ export REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" export MATRIX_PUBLIC_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" export ICONS_PUBLIC_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" -# Internal service-to-service wiring is plain HTTP on the standard ports. -# Exported so the repo's mise service tasks (which source -# mise-tasks/lib/env-vars.sh) target HTTP localhost rather than the default -# https://localhost:4201 — there is no dev cert here, so HTTPS would fail. -export REALM_BASE_URL="http://localhost:4201" +# Internal wiring runs over HTTPS, matching the repo's standard dev stack. +# env-vars.sh (sourced by `mise activate` below and by every mise service) +# detects the self-signed cert generated in setup.sh and sets +# REALM_SERVER_TLS_CERT_FILE + HOST_URL=https + REALM_BASE_URL=https +# automatically — we deliberately do NOT override those to http, because the +# prerender pipeline is HTTPS-only and a mismatch yields ERR_SSL_PROTOCOL_ERROR. +# Matrix/icons stay http (Synapse/icons don't terminate TLS locally). export MATRIX_URL="http://localhost:8008" -export MATRIX_URL_VAL="http://localhost:8008" export ICONS_URL="http://localhost:4206" export PGPORT=5435 export PGDATABASE=boxel +# Trust the self-signed cert in Node clients (the realm server's distURL fetch, +# the worker's realm reads). env-vars.sh only wires this via mkcert, which we +# don't use, so point it at the cert directly. Inherited by the mise services. +export NODE_EXTRA_CA_CERTS="$HOME/.local/share/boxel/dev-certs/localhost.pem" + # Mount-and-serve: skip the from-scratch boot index so readiness doesn't # block on a full index of every bootstrap realm; cards prerender on demand. export REALM_SERVER_SKIP_BOOT_INDEX=true -# The realm server's distURL and the prerenderer both target the local host -# over plain HTTP (no dev cert here). main.ts defaults distURL to -# https://localhost:4200, which would fail against the http vite host, so -# pin it explicitly. The mise prerender task honours this via ${HOST_URL:-…}. -export HOST_URL="http://localhost:4200" # Give the prerender's puppeteer standby probe headroom for vite's cold start. export PRERENDER_STANDBY_TIMEOUT_MS="${PRERENDER_STANDBY_TIMEOUT_MS:-120000}" @@ -65,11 +67,13 @@ gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE # ── Host app (vite) ── # Started first because it's the slowest to warm and both the realm server -# (distURL smoke test) and the prerenderer depend on it. It runs with the -# PUBLIC realm/matrix/icons URLs so prerendered output references the -# Codespace's public hostnames (which the S3 reviewer host resolves), even -# though it is itself served on plain-HTTP localhost:4200. -echo "==> Starting host app (vite) on http://localhost:4200..." +# (distURL smoke test) and the prerenderer depend on it. vite reads the +# REALM_SERVER_TLS_CERT_FILE that env-vars.sh exported (the self-signed cert) +# and serves HTTPS on localhost:4200, matching the prerender's expectation. +# It runs with the PUBLIC realm/matrix/icons URLs so prerendered output +# references the Codespace's public hostnames (which the S3 reviewer host +# resolves). +echo "==> Starting host app (vite) on https://localhost:4200..." ( cd packages/host REALM_SERVER_DOMAIN="${REALM_SERVER_URL}/" \ @@ -108,8 +112,8 @@ pnpm --dir=packages/realm-server run start:worker-development & # standby probe loads the host) and the realm server (its distURL smoke # test fetches the host). vite cold-starts the full host dep graph, so # allow generous time; a missing host is fatal to the realm server. -echo "==> Waiting for host app at http://localhost:4200 (vite cold start)..." -if ! timeout 420 bash -c 'until curl -sf http://localhost:4200 >/dev/null 2>&1; do +echo "==> Waiting for host app at https://localhost:4200 (vite cold start)..." +if ! timeout 420 bash -c 'until curl -ksf https://localhost:4200 >/dev/null 2>&1; do kill -0 '"$HOST_PID"' 2>/dev/null || { echo "host vite process died; see /tmp/host-vite.log"; exit 1; } sleep 3 done'; then @@ -134,11 +138,13 @@ if [ -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]; then fi # ── Realm server ── -# Launched by hand (not via mise) to run plain HTTP and skip the dev-cert -# requirement. toUrls are the public Codespace URLs so the S3 host resolves -# realms back to this backend. Realm layout matches mise-tasks/services/ -# realm-server: base, catalog, skills, openrouter (experiments / homepage / -# submission / software-factory are skipped to keep the preview lean). +# Launched by hand (not via mise) so we can point its toUrls at the public +# Codespace URLs (so the S3 host resolves realms back to this backend) while +# everything else matches mise-tasks/services/realm-server. It inherits +# REALM_SERVER_TLS_CERT_FILE / HOST_URL (https) / NODE_EXTRA_CA_CERTS from the +# env-vars.sh that `mise activate` sourced, so it serves HTTPS and trusts the +# vite host's cert. Realm layout: base, catalog, skills, openrouter +# (experiments / homepage / submission / software-factory skipped to stay lean). echo "==> Starting realm server..." SKIP_EXPERIMENTS=true \ SKIP_BOXEL_HOMEPAGE=true \ @@ -188,9 +194,11 @@ REALM_SERVER_SKIP_BOOT_INDEX=true \ REALM_PID=$! # ── Wait for realm server readiness ── +# The realm server now serves HTTPS on 4201 (self-signed cert), so probe with +# https + -k. echo "==> Waiting for realm server to be ready..." timeout 300 bash -c \ - 'until curl -sf "http://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ + 'until curl -ksf "https://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ || echo "Warning: realm server readiness check timed out after 5 minutes" # ── Record this Codespace as the preview target ── From 08243f8776fe3a2d63d7964edc5fc1eafbe9778a Mon Sep 17 00:00:00 2001 From: Codespace Preview Date: Wed, 17 Jun 2026 22:52:03 +0000 Subject: [PATCH 15/37] ci: record Codespace preview target --- .devcontainer/codespace-target.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/codespace-target.env b/.devcontainer/codespace-target.env index 7a77345f2e..75baca6c0b 100644 --- a/.devcontainer/codespace-target.env +++ b/.devcontainer/codespace-target.env @@ -1,5 +1,5 @@ # Written by .devcontainer/start-services.sh when a Codespace boots. # The codespaces-preview workflow reads this to point the host build at this # Codespace's forwarded backend services. Safe to delete; do not merge to main. -CODESPACE_NAME=didactic-space-funicular-gwgp7vq43w9jx +CODESPACE_NAME=sturdy-doodle-wjw4g6rjh96w9 CODESPACE_FORWARDING_DOMAIN=app.github.dev From cf9b17ffd287a7efd0ff19b7edb42291dbcf51d4 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 17:58:39 -0500 Subject: [PATCH 16/37] ci: prebuild a static host instead of vite dev for the Codespace preview vite dev's multi-minute cold-start compile on the constrained Codespace blew the readiness timeouts: /_standby navigations timed out, so the prerender manager never registered a worker, so the worker manager never started, so the realm server's hardcoded 30s waitForWorkerManager failed and it stopped. Build the host once into a static dist in setup.sh (public Codespace URLs baked in) and serve it with vite preview, which starts in seconds. Reorder start-services so the prerender precedes the worker and the realm server launches only after the worker manager reports ready, so its 30s internal wait succeeds immediately. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/setup.sh | 30 +++++++++++++++ .devcontainer/start-services.sh | 66 ++++++++++++++++----------------- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 151a0d0c8a..260e0761eb 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -54,6 +54,36 @@ if [ ! -f "$CERT_DIR/localhost.pem" ]; then -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" fi +# Build the host app once into a static dist. The realm server requires a +# reachable host (it fetches distURL at startup) and the prerenderer renders +# cards against it. Serving a prebuilt dist (vite preview) avoids vite dev's +# multi-minute cold-start compile, whose slowness blew the prerender/worker/ +# realm readiness timeouts. Built with the PUBLIC Codespace URLs so the +# served app and prerendered output reference hostnames the S3 reviewer host +# resolves. start-services.sh serves this dist on https://localhost:4200. +if [ -n "${CODESPACE_NAME:-}" ]; then + FWD_DOMAIN="${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" + REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" + echo "==> Building host app (static dist) pointed at ${REALM_SERVER_URL}..." + # boxel-ui addon must be built before the host build can resolve it + # (the dev server does this inline; the build script does not). + sh packages/boxel-ui/addon/bin/conditional-build.sh + ( + cd packages/host + REALM_SERVER_DOMAIN="${REALM_SERVER_URL}/" \ + RESOLVED_BASE_REALM_URL="${REALM_SERVER_URL}/base/" \ + RESOLVED_CATALOG_REALM_URL="${REALM_SERVER_URL}/catalog/" \ + RESOLVED_SKILLS_REALM_URL="${REALM_SERVER_URL}/skills/" \ + RESOLVED_OPENROUTER_REALM_URL="${REALM_SERVER_URL}/openrouter/" \ + MATRIX_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" \ + MATRIX_SERVER_NAME=localhost \ + ICONS_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" \ + pnpm build + ) +else + echo "==> CODESPACE_NAME unset; skipping host build (not in a Codespace)." +fi + # Database schema is created on demand: infra:ensure-pg starts the boxel-pg # container and creates the databases, and the realm server runs with # --migrateDB to apply migrations. Both happen in start-services.sh. diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index ca36e65e6a..9b7ec714f3 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -65,27 +65,14 @@ export PRERENDER_STANDBY_TIMEOUT_MS="${PRERENDER_STANDBY_TIMEOUT_MS:-120000}" echo "==> Making forwarded ports public..." gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true -# ── Host app (vite) ── -# Started first because it's the slowest to warm and both the realm server -# (distURL smoke test) and the prerenderer depend on it. vite reads the -# REALM_SERVER_TLS_CERT_FILE that env-vars.sh exported (the self-signed cert) -# and serves HTTPS on localhost:4200, matching the prerender's expectation. -# It runs with the PUBLIC realm/matrix/icons URLs so prerendered output -# references the Codespace's public hostnames (which the S3 reviewer host -# resolves). -echo "==> Starting host app (vite) on https://localhost:4200..." -( - cd packages/host - REALM_SERVER_DOMAIN="${REALM_SERVER_URL}/" \ - RESOLVED_BASE_REALM_URL="${REALM_SERVER_URL}/base/" \ - RESOLVED_CATALOG_REALM_URL="${REALM_SERVER_URL}/catalog/" \ - RESOLVED_SKILLS_REALM_URL="${REALM_SERVER_URL}/skills/" \ - RESOLVED_OPENROUTER_REALM_URL="${REALM_SERVER_URL}/openrouter/" \ - MATRIX_URL="${MATRIX_PUBLIC_URL}" \ - MATRIX_SERVER_NAME=localhost \ - ICONS_URL="${ICONS_PUBLIC_URL}" \ - exec node scripts/vite-serve.js -) > /tmp/host-vite.log 2>&1 & +# ── Host app (prebuilt static dist, served via vite preview) ── +# The dist is built in setup.sh with the public Codespace URLs baked in; +# serve-dist.js (vite preview) reads REALM_SERVER_TLS_CERT_FILE from +# env-vars.sh and serves it over HTTPS on localhost:4200. Serving a prebuilt +# dist (vs vite dev) starts in seconds, so the prerender's /_standby probe +# and the realm server's distURL smoke test don't blow their timeouts. +echo "==> Serving prebuilt host app on https://localhost:4200..." +(cd packages/host && exec node scripts/serve-dist.js) > /tmp/host-vite.log 2>&1 & HOST_PID=$! # ── Postgres (boxel-pg container) + databases ── @@ -101,30 +88,41 @@ SYNAPSE_PID=$! echo "==> Starting SMTP server..." (cd packages/matrix && pnpm assert-smtp-running) & -# ── Icons / worker ── +# ── Icons ── echo "==> Starting icons server..." pnpm --dir=packages/realm-server run start:icons & -echo "==> Starting worker..." -pnpm --dir=packages/realm-server run start:worker-development & - -# Wait for the host app before starting the prerenderer (its puppeteer -# standby probe loads the host) and the realm server (its distURL smoke -# test fetches the host). vite cold-starts the full host dep graph, so -# allow generous time; a missing host is fatal to the realm server. -echo "==> Waiting for host app at https://localhost:4200 (vite cold start)..." -if ! timeout 420 bash -c 'until curl -ksf https://localhost:4200 >/dev/null 2>&1; do - kill -0 '"$HOST_PID"' 2>/dev/null || { echo "host vite process died; see /tmp/host-vite.log"; exit 1; } - sleep 3 +# Wait for the host before the prerender (its puppeteer /_standby probe loads +# the host) and the realm server (its distURL smoke test fetches it). With a +# prebuilt dist this is quick, but allow headroom; a missing host is fatal. +echo "==> Waiting for host app at https://localhost:4200..." +if ! timeout 180 bash -c 'until curl -ksf https://localhost:4200 >/dev/null 2>&1; do + kill -0 '"$HOST_PID"' 2>/dev/null || { echo "host process died; see /tmp/host-vite.log"; exit 1; } + sleep 2 done'; then echo "Warning: host app not reachable; realm smoke test and prerender will likely fail. See /tmp/host-vite.log" fi -# ── Prerender (needs the host to be up) ── +# ── Prerender (needs the host) then worker (depends on the prerender) ── +# Ordered deliberately: the worker manager only becomes ready after the +# prerender registers a worker, and the realm server's waitForWorkerManager +# is a hardcoded 30s — so the realm must start only once the worker manager +# is already up (see the wait below), and the prerender must precede the worker. echo "==> Starting prerender services..." pnpm --dir=packages/realm-server run start:prerender-manager-dev & pnpm --dir=packages/realm-server run start:prerender-dev & +echo "==> Starting worker..." +pnpm --dir=packages/realm-server run start:worker-development & + +# Gate on the worker manager's own readiness signal before launching the +# realm server, so its 30s internal wait succeeds immediately. The manager +# reports {"ready":true} once it has a registered worker (which transitively +# requires the prerender + host to be up). +echo "==> Waiting for worker manager at http://localhost:4210 to be ready..." +timeout 900 bash -c 'until curl -sf http://localhost:4210/ 2>/dev/null | grep -q "\"ready\":true"; do sleep 3; done' \ + || echo "Warning: worker manager not ready after 900s; realm server will likely fail to start" + # Wait for Synapse before starting the realm server (it registers the # realm_server Matrix user during boot). wait $SYNAPSE_PID || true From 3333b047c863868cfe48260318daf43b1e9ed55c Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 18:46:19 -0500 Subject: [PATCH 17/37] ci: add sshd feature so the Codespace is reachable over gh codespace ssh Successfully-built containers had no SSH server, so `gh codespace ssh` only ever connected to failed/recovery containers. Add the sshd devcontainer feature so the running backend can be reached and driven over SSH. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a0cb6e2364..0c13082e6d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,7 +14,8 @@ "ghcr.io/devcontainers/features/python:1": { "version": "3.11" }, - "ghcr.io/devcontainers/features/github-cli:1": {} + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/sshd:1": {} }, "forwardPorts": [4201, 4206, 8008], From cfcf9802ce197ec31c82c263f8cfb95accc6e351 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 17 Jun 2026 22:42:14 -0500 Subject: [PATCH 18/37] ci: reuse the CI/S3 host build instead of building in the Codespace The local host build was costly and fragile (dev build's /_standby loaded too slowly for the prerender's 30s probe; the production build errored and serve-dist 404'd). The codespaces-preview workflow already builds a production host with this Codespace's URLs and deploys it to S3/CloudFront, which loads /_standby in ~0.15s. Drop the local build from setup.sh. start-services.sh now records + pushes the target file up front (triggering the CI rebuild), then points the realm server's distURL (HOST_URL) and the prerenderer (BOXEL_HOST_URL) at the S3 host, waiting for it to be live before starting them. Realm boots against the already-deployed S3 host; the fresh rebuild for this Codespace lands in the background. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/setup.sh | 35 ++-------- .devcontainer/start-services.sh | 114 +++++++++++++++----------------- 2 files changed, 59 insertions(+), 90 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 260e0761eb..fb7439956e 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -54,35 +54,12 @@ if [ ! -f "$CERT_DIR/localhost.pem" ]; then -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" fi -# Build the host app once into a static dist. The realm server requires a -# reachable host (it fetches distURL at startup) and the prerenderer renders -# cards against it. Serving a prebuilt dist (vite preview) avoids vite dev's -# multi-minute cold-start compile, whose slowness blew the prerender/worker/ -# realm readiness timeouts. Built with the PUBLIC Codespace URLs so the -# served app and prerendered output reference hostnames the S3 reviewer host -# resolves. start-services.sh serves this dist on https://localhost:4200. -if [ -n "${CODESPACE_NAME:-}" ]; then - FWD_DOMAIN="${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" - REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" - echo "==> Building host app (static dist) pointed at ${REALM_SERVER_URL}..." - # boxel-ui addon must be built before the host build can resolve it - # (the dev server does this inline; the build script does not). - sh packages/boxel-ui/addon/bin/conditional-build.sh - ( - cd packages/host - REALM_SERVER_DOMAIN="${REALM_SERVER_URL}/" \ - RESOLVED_BASE_REALM_URL="${REALM_SERVER_URL}/base/" \ - RESOLVED_CATALOG_REALM_URL="${REALM_SERVER_URL}/catalog/" \ - RESOLVED_SKILLS_REALM_URL="${REALM_SERVER_URL}/skills/" \ - RESOLVED_OPENROUTER_REALM_URL="${REALM_SERVER_URL}/openrouter/" \ - MATRIX_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" \ - MATRIX_SERVER_NAME=localhost \ - ICONS_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" \ - pnpm build - ) -else - echo "==> CODESPACE_NAME unset; skipping host build (not in a Codespace)." -fi +# The host app is NOT built here. The realm server requires a reachable host +# (it fetches distURL at startup) and the prerenderer renders cards against +# it, but rather than build a second copy in the Codespace, start-services.sh +# points both at the host the codespaces-preview workflow already builds and +# deploys to S3 (a production, CloudFront-served bundle that loads far faster +# than a local dev build). # Database schema is created on demand: infra:ensure-pg starts the boxel-pg # container and creates the databases, and the realm server runs with diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 9b7ec714f3..4898b0e7c9 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -61,19 +61,43 @@ export REALM_SERVER_SKIP_BOOT_INDEX=true # Give the prerender's puppeteer standby probe headroom for vite's cold start. export PRERENDER_STANDBY_TIMEOUT_MS="${PRERENDER_STANDBY_TIMEOUT_MS:-120000}" -# ── Make forwarded ports public so the S3 preview can reach them ── +# ── Make forwarded ports public so the S3 host can reach this backend ── echo "==> Making forwarded ports public..." gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true -# ── Host app (prebuilt static dist, served via vite preview) ── -# The dist is built in setup.sh with the public Codespace URLs baked in; -# serve-dist.js (vite preview) reads REALM_SERVER_TLS_CERT_FILE from -# env-vars.sh and serves it over HTTPS on localhost:4200. Serving a prebuilt -# dist (vs vite dev) starts in seconds, so the prerender's /_standby probe -# and the realm server's distURL smoke test don't blow their timeouts. -echo "==> Serving prebuilt host app on https://localhost:4200..." -(cd packages/host && exec node scripts/serve-dist.js) > /tmp/host-vite.log 2>&1 & -HOST_PID=$! +# ── Record this Codespace as the preview target (triggers the CI host build) ── +# Done up-front: the codespaces-preview workflow builds the host with THIS +# Codespace's URLs and deploys it to S3, and the realm server (distURL) + the +# prerenderer point at that S3 host. Pushing the target file rebuilds it for +# this Codespace; the already-deployed S3 host stays usable in the meantime. +echo "==> Recording Codespace target for the preview workflow..." +TARGET_FILE=".devcontainer/codespace-target.env" +cat > "$TARGET_FILE" </dev/null + git push origin "HEAD:${BRANCH_NAME}" \ + || echo "Warning: could not push Codespace target; the preview build was not retriggered." +fi + +# The host the realm + prerenderer use is the CI/S3 build. The bucket prefix +# is the branch name sanitized exactly as the workflow does (PR_BRANCH_NAME). +PR_BRANCH_NAME="$(echo "$BRANCH_NAME" | tr _ - | tr '[:upper:]' '[:lower:]' | sed -e 's/-$//' | sed -e 's/[^a-z0-9\-]//g' | cut -c1-60)" +export BOXEL_HOST_URL="https://${PR_BRANCH_NAME}.boxel-host-preview.stack.cards" +echo "==> Host (for realm distURL + prerender): ${BOXEL_HOST_URL}" # ── Postgres (boxel-pg container) + databases ── echo "==> Ensuring Postgres is running..." @@ -92,16 +116,13 @@ echo "==> Starting SMTP server..." echo "==> Starting icons server..." pnpm --dir=packages/realm-server run start:icons & -# Wait for the host before the prerender (its puppeteer /_standby probe loads -# the host) and the realm server (its distURL smoke test fetches it). With a -# prebuilt dist this is quick, but allow headroom; a missing host is fatal. -echo "==> Waiting for host app at https://localhost:4200..." -if ! timeout 180 bash -c 'until curl -ksf https://localhost:4200 >/dev/null 2>&1; do - kill -0 '"$HOST_PID"' 2>/dev/null || { echo "host process died; see /tmp/host-vite.log"; exit 1; } - sleep 2 - done'; then - echo "Warning: host app not reachable; realm smoke test and prerender will likely fail. See /tmp/host-vite.log" -fi +# Wait for the S3 host to be live before the prerender (its puppeteer /_standby +# probe loads it) and the realm server (its distURL smoke test fetches it). +# Usually already live from a prior build; the long timeout covers a +# first-ever build that has to wait for CI. +echo "==> Waiting for S3 host at ${BOXEL_HOST_URL}/_standby..." +timeout 900 bash -c 'until curl -sf "'"$BOXEL_HOST_URL"'/_standby" >/dev/null 2>&1; do sleep 5; done' \ + || echo "Warning: S3 host not reachable; realm smoke test and prerender will likely fail." # ── Prerender (needs the host) then worker (depends on the prerender) ── # Ordered deliberately: the worker manager only becomes ready after the @@ -137,11 +158,11 @@ fi # ── Realm server ── # Launched by hand (not via mise) so we can point its toUrls at the public -# Codespace URLs (so the S3 host resolves realms back to this backend) while -# everything else matches mise-tasks/services/realm-server. It inherits -# REALM_SERVER_TLS_CERT_FILE / HOST_URL (https) / NODE_EXTRA_CA_CERTS from the -# env-vars.sh that `mise activate` sourced, so it serves HTTPS and trusts the -# vite host's cert. Realm layout: base, catalog, skills, openrouter +# Codespace URLs (so the S3 host resolves realms back to this backend) and its +# distURL (HOST_URL) at the CI/S3 host, while everything else matches +# mise-tasks/services/realm-server. It inherits REALM_SERVER_TLS_CERT_FILE / +# NODE_EXTRA_CA_CERTS from the env-vars.sh that `mise activate` sourced, so it +# serves HTTPS on 4201. Realm layout: base, catalog, skills, openrouter # (experiments / homepage / submission / software-factory skipped to stay lean). echo "==> Starting realm server..." SKIP_EXPERIMENTS=true \ @@ -157,6 +178,7 @@ REALM_SERVER_SECRET_SEED="mum's the word" \ REALM_SECRET_SEED="shhh! it's a secret" \ GRAFANA_SECRET="shhh! it's a secret" \ LOW_CREDIT_THRESHOLD="${LOW_CREDIT_THRESHOLD:-2000}" \ +HOST_URL="$BOXEL_HOST_URL" \ MATRIX_URL=http://localhost:8008 \ REALM_SERVER_MATRIX_USERNAME=realm_server \ ENABLE_FILE_WATCHER=true \ @@ -199,34 +221,6 @@ timeout 300 bash -c \ 'until curl -ksf "https://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ || echo "Warning: realm server readiness check timed out after 5 minutes" -# ── Record this Codespace as the preview target ── -# The codespaces-preview workflow triggers on pushes to this branch and reads -# the committed target file to point the host build at this Codespace's -# forwarded backend. Committing + pushing it triggers the first build; every -# later code push then rebuilds against the same backend. -echo "==> Recording Codespace target for the preview workflow..." -TARGET_FILE=".devcontainer/codespace-target.env" -cat > "$TARGET_FILE" </dev/null - git push origin "HEAD:${BRANCH_NAME}" \ - || echo "Warning: could not push Codespace target; the preview build was not triggered." -fi - echo "" echo "============================================" echo " Backend services running!" @@ -234,14 +228,12 @@ echo "" echo " Realm server: ${REALM_SERVER_URL}" echo " Matrix: ${MATRIX_PUBLIC_URL}" echo " Icons: ${ICONS_PUBLIC_URL}" -echo "" -echo " Host preview build triggered — check the" -echo " PR for a preview link once it completes." +echo " Host preview: ${BOXEL_HOST_URL}" echo "============================================" -# This script is launched detached (nohup ... &) from postStartCommand so the -# Codespace lifecycle hook returns immediately — a blocking postStartCommand -# wedges the Codespaces agent and SSH never comes up. Blocking on the realm -# server here keeps this script alive as the parent of all the backgrounded -# services for the life of the Codespace. Output goes to /tmp/start-services.log. +# This script is launched detached from postStartCommand so the Codespace +# lifecycle hook returns immediately — a blocking postStartCommand wedges the +# Codespaces agent. Blocking on the realm server here keeps this script alive +# as the parent of all the backgrounded services for the life of the +# Codespace. Output goes to /tmp/start-services.log. wait "$REALM_PID" From a658cb95ed83145254040c16599b04f4161e69f7 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 07:44:19 -0500 Subject: [PATCH 19/37] ci: give the Codespace realm server its public serverURL serverURL defaults to https://localhost:4201, and the realm injects it into the host page as realmServerURL, so a browser visiting the realm is sent to localhost. Pass --serverURL= so the served host app talks to the forwarded realm instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 4898b0e7c9..c5833d2d4e 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -186,6 +186,7 @@ REALM_SERVER_SKIP_BOOT_INDEX=true \ pnpm --dir=packages/realm-server exec ts-node \ --transpileOnly main \ --port=4201 \ + --serverURL="${REALM_SERVER_URL}" \ --matrixURL=http://localhost:8008 \ --realmsRootPath=./realms/codespaces \ --prerendererUrl=http://localhost:4222 \ From 2b2b03fdf3783900d8847ac58dc401486e96dac6 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 07:50:58 -0500 Subject: [PATCH 20/37] ci: rebase before pushing the Codespace target so it isn't silently dropped A fresh Codespace's target-file push was rejected whenever origin had advanced, so the S3 host stayed built for an earlier Codespace (dead Matrix/icons URLs). Rebase onto origin before writing + committing the target file so the push fast-forwards and the host rebuilds for the live Codespace. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index c5833d2d4e..f0beef23c2 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -72,6 +72,15 @@ gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE # this Codespace; the already-deployed S3 host stays usable in the meantime. echo "==> Recording Codespace target for the preview workflow..." TARGET_FILE=".devcontainer/codespace-target.env" +BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" + +# Incorporate any commits pushed since this Codespace was created, so the +# target-file commit fast-forwards instead of being rejected (which would +# silently leave the S3 host built for a previous Codespace). Rebase BEFORE +# writing the target file so there's no same-file conflict with origin's copy. +git pull --rebase --autostash origin "$BRANCH_NAME" 2>/dev/null \ + || echo "Warning: could not rebase onto origin; target push may be rejected." + cat > "$TARGET_FILE" </dev/null - git push origin "HEAD:${BRANCH_NAME}" \ - || echo "Warning: could not push Codespace target; the preview build was not retriggered." + if git push origin "HEAD:${BRANCH_NAME}"; then + echo "Pushed Codespace target; the preview host is rebuilding for this Codespace." + else + echo "Warning: could not push Codespace target; the preview build was not retriggered." + fi fi # The host the realm + prerenderer use is the CI/S3 build. The bucket prefix From fc3099a17aec67e783573042f8026b46068c4045 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 08:51:49 -0500 Subject: [PATCH 21/37] ci: log into the Codespace's own Synapse via realm-injected config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The realm server already rewrites the host's Ember config at serve time (serve-index.ts: matrixURL, matrixServerName, realmServerURL, resolved realm URLs), so the preview's entry point should be the realm URL — it injects this Codespace's endpoints, no per-Codespace host rebuild needed. Two fixes so a reviewer can actually log in: - serve-index injects matrixClient's backend URL (localhost in a Codespace, unreachable from a browser). Add a RESOLVED_MATRIX_URL env override for the browser-facing Matrix URL, mirroring how serverURL is already separate from the local bind. No-op when unset (prod/staging). - start-services.sh sets RESOLVED_MATRIX_URL + MATRIX_SERVER_NAME=localhost on the realm, and seeds the dev user/password on this Codespace's Synapse so the preview is loginnable. The banner now points reviewers at the realm URL with user/password. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 23 +++++++++++++++---- packages/realm-server/handlers/serve-index.ts | 10 +++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index f0beef23c2..d024b6aa33 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -167,6 +167,14 @@ if [ -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]; then export MATRIX_REGISTRATION_SHARED_SECRET fi +# Seed a reviewer login (user / password) on this Codespace's own Synapse so +# the preview is loginnable. This is the same dev user local development uses; +# it's only as exposed as the forwarded Matrix port (see the abuse note in the +# PR / README — gate port visibility if that matters for a given preview). +echo "==> Seeding reviewer Matrix user (user/password)..." +(cd packages/matrix && MATRIX_URL=http://localhost:8008 pnpm register-test-user) \ + || echo "Note: reviewer user seeding skipped (it may already exist)." + # ── Realm server ── # Launched by hand (not via mise) so we can point its toUrls at the public # Codespace URLs (so the S3 host resolves realms back to this backend) and its @@ -191,6 +199,8 @@ GRAFANA_SECRET="shhh! it's a secret" \ LOW_CREDIT_THRESHOLD="${LOW_CREDIT_THRESHOLD:-2000}" \ HOST_URL="$BOXEL_HOST_URL" \ MATRIX_URL=http://localhost:8008 \ +RESOLVED_MATRIX_URL="${MATRIX_PUBLIC_URL}" \ +MATRIX_SERVER_NAME=localhost \ REALM_SERVER_MATRIX_USERNAME=realm_server \ ENABLE_FILE_WATCHER=true \ REALM_SERVER_SKIP_BOOT_INDEX=true \ @@ -237,10 +247,15 @@ echo "" echo "============================================" echo " Backend services running!" echo "" -echo " Realm server: ${REALM_SERVER_URL}" -echo " Matrix: ${MATRIX_PUBLIC_URL}" -echo " Icons: ${ICONS_PUBLIC_URL}" -echo " Host preview: ${BOXEL_HOST_URL}" +echo " Open the preview at the REALM SERVER URL — it serves the host app" +echo " with this Codespace's endpoints injected (see serve-index.ts):" +echo "" +echo " ${REALM_SERVER_URL}" +echo "" +echo " Log in with: user / password" +echo "" +echo " (Assets are served from ${BOXEL_HOST_URL};" +echo " Matrix ${MATRIX_PUBLIC_URL})" echo "============================================" # This script is launched detached from postStartCommand so the Codespace diff --git a/packages/realm-server/handlers/serve-index.ts b/packages/realm-server/handlers/serve-index.ts index 719effaf05..c68fb7caa1 100644 --- a/packages/realm-server/handlers/serve-index.ts +++ b/packages/realm-server/handlers/serve-index.ts @@ -122,7 +122,15 @@ export function createServeIndex(deps: ServeIndexDeps): ServeIndexHandlers { config = merge({}, config, { hostsOwnAssets: false, assetsURL: assetsURL.href, - matrixURL: matrixClient.matrixURL.href.replace(/\/$/, ''), + // The browser-facing Matrix URL can differ from the realm + // server's own backend connection (e.g. in a Codespace the + // backend talks to Synapse on localhost while the browser must + // use the public forwarded URL). RESOLVED_MATRIX_URL lets a + // deployment inject the browser-facing URL without rerouting the + // realm's own Matrix client. Falls back to the backend URL. + matrixURL: ( + process.env.RESOLVED_MATRIX_URL ?? matrixClient.matrixURL.href + ).replace(/\/$/, ''), matrixServerName: process.env.MATRIX_SERVER_NAME || matrixClient.matrixURL.hostname, realmServerURL: serverURL.href, From 0460129612e7776451aa66d13e4caad303575eef Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 09:00:20 -0500 Subject: [PATCH 22/37] ci: build the Codespace preview host generically (drop per-Codespace rebuild) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The realm server injects this Codespace's endpoints into the host config at serve time, so the host build no longer needs Codespace-specific URLs baked in. Build it generically once per code change and deploy to S3; the realm fetches that bundle as distURL and serves it config-injected. Removes the target-file push, the rebase-before-push divergence handling, and the codespace-target.env file — the whole per-Codespace rebuild dance that kept the S3 host pinned to a stale/dead Codespace. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/codespace-target.env | 5 - .devcontainer/start-services.sh | 47 ++------ .github/workflows/codespaces-preview.yml | 131 +++-------------------- 3 files changed, 20 insertions(+), 163 deletions(-) delete mode 100644 .devcontainer/codespace-target.env diff --git a/.devcontainer/codespace-target.env b/.devcontainer/codespace-target.env deleted file mode 100644 index 75baca6c0b..0000000000 --- a/.devcontainer/codespace-target.env +++ /dev/null @@ -1,5 +0,0 @@ -# Written by .devcontainer/start-services.sh when a Codespace boots. -# The codespaces-preview workflow reads this to point the host build at this -# Codespace's forwarded backend services. Safe to delete; do not merge to main. -CODESPACE_NAME=sturdy-doodle-wjw4g6rjh96w9 -CODESPACE_FORWARDING_DOMAIN=app.github.dev diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index d024b6aa33..162cf3276c 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -65,47 +65,14 @@ export PRERENDER_STANDBY_TIMEOUT_MS="${PRERENDER_STANDBY_TIMEOUT_MS:-120000}" echo "==> Making forwarded ports public..." gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true -# ── Record this Codespace as the preview target (triggers the CI host build) ── -# Done up-front: the codespaces-preview workflow builds the host with THIS -# Codespace's URLs and deploys it to S3, and the realm server (distURL) + the -# prerenderer point at that S3 host. Pushing the target file rebuilds it for -# this Codespace; the already-deployed S3 host stays usable in the meantime. -echo "==> Recording Codespace target for the preview workflow..." -TARGET_FILE=".devcontainer/codespace-target.env" +# ── Host bundle (generic CI/S3 build; config injected at serve time) ── +# No per-Codespace rebuild or target-file push: the codespaces-preview +# workflow builds the host generically and deploys it to S3, and the realm +# server rewrites the Ember config (Matrix / realm-server / resolved-realm +# URLs) into it at serve time for THIS Codespace (serve-index.ts). The realm +# fetches this bundle as its distURL and serves it; the prerenderer renders +# against it. Bucket prefix = the branch name sanitized as the workflow does. BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" - -# Incorporate any commits pushed since this Codespace was created, so the -# target-file commit fast-forwards instead of being rejected (which would -# silently leave the S3 host built for a previous Codespace). Rebase BEFORE -# writing the target file so there's no same-file conflict with origin's copy. -git pull --rebase --autostash origin "$BRANCH_NAME" 2>/dev/null \ - || echo "Warning: could not rebase onto origin; target push may be rejected." - -cat > "$TARGET_FILE" </dev/null - if git push origin "HEAD:${BRANCH_NAME}"; then - echo "Pushed Codespace target; the preview host is rebuilding for this Codespace." - else - echo "Warning: could not push Codespace target; the preview build was not retriggered." - fi -fi - -# The host the realm + prerenderer use is the CI/S3 build. The bucket prefix -# is the branch name sanitized exactly as the workflow does (PR_BRANCH_NAME). PR_BRANCH_NAME="$(echo "$BRANCH_NAME" | tr _ - | tr '[:upper:]' '[:lower:]' | sed -e 's/-$//' | sed -e 's/[^a-z0-9\-]//g' | cut -c1-60)" export BOXEL_HOST_URL="https://${PR_BRANCH_NAME}.boxel-host-preview.stack.cards" echo "==> Host (for realm distURL + prerender): ${BOXEL_HOST_URL}" diff --git a/.github/workflows/codespaces-preview.yml b/.github/workflows/codespaces-preview.yml index 9aa21537c5..1f175d6f24 100644 --- a/.github/workflows/codespaces-preview.yml +++ b/.github/workflows/codespaces-preview.yml @@ -1,115 +1,47 @@ name: Codespaces Preview +# Builds the boxel-host app and deploys it to S3. The build is GENERIC — it +# does not bake in any Codespace's URLs, because the realm server rewrites the +# host's Ember config (Matrix / realm-server / resolved-realm URLs) at serve +# time (see packages/realm-server/handlers/serve-index.ts). A Codespace's +# start-services.sh points the realm's distURL at this S3 bundle and serves it +# with this Codespace's endpoints injected, so one generic build per code +# change serves every Codespace — no per-Codespace rebuild. on: - # Pushes to the preview branch rebuild the host preview against whatever - # Codespace is currently recorded in .devcontainer/codespace-target.env. - # That file is written + committed by .devcontainer/start-services.sh when - # a Codespace boots, so the Codespace's own startup push triggers the first - # build and every subsequent code push rebuilds against the same backend. - # Add other branches here to enable Codespace previews on them. push: branches: - codespaces-10255 - # Manual fallback — supply the Codespace name directly. - workflow_dispatch: - inputs: - codespace_name: - description: "Codespace name (used to derive forwarded port URLs)" - required: true - type: string - forwarding_domain: - description: "Codespaces port-forwarding domain" - required: false - default: app.github.dev - type: string + workflow_dispatch: {} permissions: contents: read - issues: write - checks: write - pull-requests: write id-token: write - statuses: write jobs: deploy-codespaces-preview: - name: Build and deploy host preview for Codespaces + name: Build and deploy the generic host bundle for Codespace previews runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Resolve Codespace target - shell: bash - run: | - # Determine the Codespace name from the dispatch input or the - # committed target file, then derive the forwarded backend URLs. - # If neither is available (e.g. a code push before any Codespace - # has booted), skip the rest of the job cleanly. - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - NAME="${{ inputs.codespace_name }}" - DOMAIN="${{ inputs.forwarding_domain }}" - elif [ -f .devcontainer/codespace-target.env ]; then - # shellcheck disable=SC1091 - . .devcontainer/codespace-target.env - NAME="${CODESPACE_NAME:-}" - DOMAIN="${CODESPACE_FORWARDING_DOMAIN:-}" - else - NAME="" - fi - DOMAIN="${DOMAIN:-app.github.dev}" - if [ -z "$NAME" ]; then - echo "No Codespace target recorded; skipping preview build." - echo "SHOULD_RUN=false" >> "$GITHUB_ENV" - exit 0 - fi - { - echo "SHOULD_RUN=true" - echo "CODESPACE_NAME=$NAME" - echo "REALM_SERVER_URL=https://$NAME-4201.$DOMAIN" - echo "MATRIX_PUBLIC_URL=https://$NAME-8008.$DOMAIN" - echo "ICONS_PUBLIC_URL=https://$NAME-4206.$DOMAIN" - } >> "$GITHUB_ENV" - - uses: ./.github/actions/init - if: env.SHOULD_RUN == 'true' - name: Configure AWS credentials - if: env.SHOULD_RUN == 'true' uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0 with: role-to-assume: arn:aws:iam::680542703984:role/boxel-host aws-region: us-east-1 - - name: Load staging app environment variables as a base - if: env.SHOULD_RUN == 'true' + - name: Load staging app environment variables (overridden at serve time) shell: bash run: cat packages/host/config/staging.env >> $GITHUB_ENV - - name: Override backend URLs to point at the Codespace - if: env.SHOULD_RUN == 'true' + - name: Set branch name for the S3 prefix shell: bash run: | - # These override the staging.env values loaded above (last write - # wins in $GITHUB_ENV) so the host build talks to this Codespace's - # forwarded backend services instead of staging. - echo "REALM_SERVER_DOMAIN=${REALM_SERVER_URL}/" >> $GITHUB_ENV - echo "RESOLVED_BASE_REALM_URL=${REALM_SERVER_URL}/base/" >> $GITHUB_ENV - echo "RESOLVED_CATALOG_REALM_URL=${REALM_SERVER_URL}/catalog/" >> $GITHUB_ENV - echo "RESOLVED_SKILLS_REALM_URL=${REALM_SERVER_URL}/skills/" >> $GITHUB_ENV - echo "RESOLVED_OPENROUTER_REALM_URL=${REALM_SERVER_URL}/openrouter/" >> $GITHUB_ENV - echo "MATRIX_URL=${MATRIX_PUBLIC_URL}" >> $GITHUB_ENV - echo "MATRIX_SERVER_NAME=localhost" >> $GITHUB_ENV - echo "ICONS_URL=${ICONS_PUBLIC_URL}" >> $GITHUB_ENV + echo "PR_BRANCH_NAME=$(echo "${GITHUB_REF_NAME}" | tr _ - | tr '[:upper:]' '[:lower:]' | sed -e 's/-$//' | sed -e 's/[^a-z0-9\-]//g' | cut -c1-60)" >> $GITHUB_ENV - - name: Set PR branch name for S3 prefix - if: env.SHOULD_RUN == 'true' - shell: bash - run: | - RAW_BRANCH="${GITHUB_REF_NAME}" - echo "PR_BRANCH_NAME=$(echo "${RAW_BRANCH}" | tr _ - | tr '[:upper:]' '[:lower:]' | sed -e 's/-$//' | sed -e 's/[^a-z0-9\-]//g' | cut -c1-60)" >> $GITHUB_ENV - - - name: Build and deploy preview - if: env.SHOULD_RUN == 'true' + - name: Build and deploy host bundle shell: bash env: S3_PREVIEW_BUCKET_NAME: boxel-host-preview.stack.cards @@ -117,40 +49,3 @@ jobs: AWS_REGION: us-east-1 AWS_CLOUDFRONT_DISTRIBUTION: EU4RGLH4EOCHJ run: pnpm deploy:boxel-host:preview-staging - - - name: Store preview URL - if: env.SHOULD_RUN == 'true' - shell: bash - run: echo "PREVIEW_HOST=https://${PR_BRANCH_NAME}.boxel-host-preview.stack.cards/" >> $GITHUB_ENV - - - name: Post status check with preview link - if: env.SHOULD_RUN == 'true' - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - HEAD_SHA: ${{ github.sha }} - run: | - curl \ - -X POST \ - -H "Authorization: token $GITHUB_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/$REPOSITORY/statuses/$HEAD_SHA" \ - -d '{"context":"Preview boxel-host codespaces","description":"Host preview connected to Codespace backend","target_url":"'"$PREVIEW_HOST"'","state":"success"}' - - - name: Find associated PR and comment - if: env.SHOULD_RUN == 'true' - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - PR_NUMBER=$(gh pr list --head "${GITHUB_REF_NAME}" --json number --jq '.[0].number' 2>/dev/null || true) - if [ -n "$PR_NUMBER" ]; then - gh pr comment "$PR_NUMBER" --body "### Codespaces Preview - - - [Host preview (connected to Codespace)]($PREVIEW_HOST) - - Realm server: ${REALM_SERVER_URL} - - Matrix: ${MATRIX_PUBLIC_URL} - - > This preview build is connected to the Codespace \`${CODESPACE_NAME}\`. The preview will stop working when the Codespace is stopped." - fi From 5a5dc38044414cd19ee6a87d24eb67f6cdf21b21 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 09:44:39 -0500 Subject: [PATCH 23/37] ci: serve the Codespace realm over plain HTTP (GitHub edge does TLS) Visiting the forwarded realm URL bounced to https://localhost:4201: GitHub's port forwarding terminates TLS at its edge and forwards plain HTTP to the backend, but the realm served HTTPS and 308-redirected the plain-HTTP request to its own https bind address. Drop the self-signed cert (the prerender uses the public S3 host now, so nothing local needs TLS): no cert -> realm serves plain HTTP -> no redirect, GitHub's edge provides the public HTTPS. Force REALM_BASE_URL + Matrix/icons to http so the worker/prerender dial the plain-HTTP realm, and probe readiness over http. serverURL/toUrls stay the public URLs, so the realm still injects public addresses into the served host page. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/setup.sh | 24 +++++------------- .devcontainer/start-services.sh | 43 ++++++++++++++------------------- 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index fb7439956e..8083abacbe 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -35,24 +35,12 @@ echo "==> Setting up catalog realm..." mise exec -- pnpm --dir=packages/catalog catalog:setup mise exec -- pnpm --dir=packages/catalog catalog:update -# The realm server, vite host and prerenderer all speak HTTPS in this repo -# (browsers only do HTTP/2 over TLS, which the prerender pipeline assumes). -# Generate a self-signed cert at the path env-vars.sh probes -# (~/.local/share/boxel/dev-certs) so it sets REALM_SERVER_TLS_CERT_FILE and -# HOST_URL=https for every mise service, and vite serves HTTPS too. The -# prerender's puppeteer ignores cert errors; Node trusts it via -# NODE_EXTRA_CA_CERTS (set in start-services.sh). mkcert isn't used — nothing -# here needs the cert in a browser/system trust store. -echo "==> Generating self-signed dev TLS cert..." -CERT_DIR="$HOME/.local/share/boxel/dev-certs" -mkdir -p "$CERT_DIR" -if [ ! -f "$CERT_DIR/localhost.pem" ]; then - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$CERT_DIR/localhost-key.pem" \ - -out "$CERT_DIR/localhost.pem" \ - -days 365 -subj "/CN=localhost" \ - -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" -fi +# No local TLS cert: the realm server serves plain HTTP, and GitHub's port +# forwarding terminates TLS at its edge (it forwards plain HTTP to the +# backend). If the realm served HTTPS it would 308-redirect the edge's +# plain-HTTP request to https://localhost:4201 — bouncing the browser to +# localhost. The prerender uses the public S3 host (already HTTPS), so +# nothing here needs a local cert. # The host app is NOT built here. The realm server requires a reachable host # (it fetches distURL at startup) and the prerenderer renders cards against diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 162cf3276c..7b31cdd491 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -37,30 +37,25 @@ export REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" export MATRIX_PUBLIC_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" export ICONS_PUBLIC_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" -# Internal wiring runs over HTTPS, matching the repo's standard dev stack. -# env-vars.sh (sourced by `mise activate` below and by every mise service) -# detects the self-signed cert generated in setup.sh and sets -# REALM_SERVER_TLS_CERT_FILE + HOST_URL=https + REALM_BASE_URL=https -# automatically — we deliberately do NOT override those to http, because the -# prerender pipeline is HTTPS-only and a mismatch yields ERR_SSL_PROTOCOL_ERROR. -# Matrix/icons stay http (Synapse/icons don't terminate TLS locally). +# Internal wiring runs over plain HTTP on the standard ports. No dev cert is +# generated (see setup.sh): the realm server serves plain HTTP and GitHub's +# port forwarding terminates TLS at its edge. Override the env-vars.sh +# defaults (which assume https://localhost) so the realm and every mise +# service agree on http — otherwise the worker/prerender would dial +# https://localhost:4201 against a plain-HTTP realm. The browser-facing +# public URLs (REALM_SERVER_URL etc., all https) are injected by the realm +# at serve time and passed to the host build separately. +export REALM_BASE_URL="http://localhost:4201" export MATRIX_URL="http://localhost:8008" +export MATRIX_URL_VAL="http://localhost:8008" export ICONS_URL="http://localhost:4206" export PGPORT=5435 export PGDATABASE=boxel -# Trust the self-signed cert in Node clients (the realm server's distURL fetch, -# the worker's realm reads). env-vars.sh only wires this via mkcert, which we -# don't use, so point it at the cert directly. Inherited by the mise services. -export NODE_EXTRA_CA_CERTS="$HOME/.local/share/boxel/dev-certs/localhost.pem" - # Mount-and-serve: skip the from-scratch boot index so readiness doesn't # block on a full index of every bootstrap realm; cards prerender on demand. export REALM_SERVER_SKIP_BOOT_INDEX=true -# Give the prerender's puppeteer standby probe headroom for vite's cold start. -export PRERENDER_STANDBY_TIMEOUT_MS="${PRERENDER_STANDBY_TIMEOUT_MS:-120000}" - # ── Make forwarded ports public so the S3 host can reach this backend ── echo "==> Making forwarded ports public..." gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true @@ -143,13 +138,12 @@ echo "==> Seeding reviewer Matrix user (user/password)..." || echo "Note: reviewer user seeding skipped (it may already exist)." # ── Realm server ── -# Launched by hand (not via mise) so we can point its toUrls at the public -# Codespace URLs (so the S3 host resolves realms back to this backend) and its -# distURL (HOST_URL) at the CI/S3 host, while everything else matches -# mise-tasks/services/realm-server. It inherits REALM_SERVER_TLS_CERT_FILE / -# NODE_EXTRA_CA_CERTS from the env-vars.sh that `mise activate` sourced, so it -# serves HTTPS on 4201. Realm layout: base, catalog, skills, openrouter -# (experiments / homepage / submission / software-factory skipped to stay lean). +# Launched by hand (not via mise) so we can point its toUrls + serverURL at +# the public Codespace URLs (so the served host + redirects use the public +# address, and the S3 host resolves realms back here) and its distURL +# (HOST_URL) at the CI/S3 host. Serves plain HTTP on 4201; GitHub's edge does +# TLS. Realm layout: base, catalog, skills, openrouter (experiments / +# homepage / submission / software-factory skipped to stay lean). echo "==> Starting realm server..." SKIP_EXPERIMENTS=true \ SKIP_BOXEL_HOMEPAGE=true \ @@ -203,11 +197,10 @@ REALM_SERVER_SKIP_BOOT_INDEX=true \ REALM_PID=$! # ── Wait for realm server readiness ── -# The realm server now serves HTTPS on 4201 (self-signed cert), so probe with -# https + -k. +# The realm server serves plain HTTP on 4201 (GitHub's edge terminates TLS). echo "==> Waiting for realm server to be ready..." timeout 300 bash -c \ - 'until curl -ksf "https://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ + 'until curl -sf "http://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ || echo "Warning: realm server readiness check timed out after 5 minutes" echo "" From 142c5025dc24d8177aeb894a9b2e8820d0818026 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 11:41:51 -0500 Subject: [PATCH 24/37] ci: proxy host assets same-origin in Codespace previews (fix CORS) The Codespace preview serves the host app from the realm server's own origin, but rewrote asset URLs to the S3 preview bucket. ES-module scripts are fetched in CORS mode, and the preview CloudFront sends no Access-Control-Allow-Origin, so every JS/CSS load was blocked. Point the host app's asset URLs at the realm origin (ASSETS_URL_OVERRIDE) so they're same-origin, and have the realm proxy /assets, /@embroider and the favicons through to the actual bundle (HOST_URL). Adds proxyAssetPaths and a hostDistURL upstream that defaults to assetsURL, so normal deployments are unaffected (there the HTML points assets straight at the host CDN and these paths are never requested from the realm). Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 9 ++ packages/realm-server/main.ts | 4 + packages/realm-server/middleware/index.ts | 130 ++++++++++++++-------- packages/realm-server/server.ts | 29 ++++- 4 files changed, 124 insertions(+), 48 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 7b31cdd491..bbe6cd31da 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -144,6 +144,14 @@ echo "==> Seeding reviewer Matrix user (user/password)..." # (HOST_URL) at the CI/S3 host. Serves plain HTTP on 4201; GitHub's edge does # TLS. Realm layout: base, catalog, skills, openrouter (experiments / # homepage / submission / software-factory skipped to stay lean). +# +# ASSETS_URL_OVERRIDE points the host app's asset URLs at the realm server's +# OWN origin instead of the S3 host. The reviewer's browser loads the app from +# the realm origin (4201), so loading ES-module scripts from the S3 host would +# be cross-origin and fail CORS (the preview bucket sends no +# Access-Control-Allow-Origin). With the override, asset URLs are same-origin +# and the realm proxies /assets, /@embroider and the favicons through to the +# S3 bundle (HOST_URL) — see proxyAssetPaths in packages/realm-server. echo "==> Starting realm server..." SKIP_EXPERIMENTS=true \ SKIP_BOXEL_HOMEPAGE=true \ @@ -154,6 +162,7 @@ NODE_NO_WARNINGS=1 \ PGPORT=5435 \ PGDATABASE=boxel \ LOG_LEVELS='*=info' \ +ASSETS_URL_OVERRIDE="${REALM_SERVER_URL}" \ REALM_SERVER_SECRET_SEED="mum's the word" \ REALM_SECRET_SEED="shhh! it's a secret" \ GRAFANA_SECRET="shhh! it's a secret" \ diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index b8db76c556..805982a8f8 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -583,6 +583,10 @@ const smokeTestHostApp = async () => { assetsURL: process.env.ASSETS_URL_OVERRIDE ? new URL(process.env.ASSETS_URL_OVERRIDE) : dist, + // The actual host bundle, used to proxy static assets when assetsURL is + // overridden to the realm's own origin (so same-origin asset requests are + // forwarded to the real bundle). Equals assetsURL when not overridden. + hostDistURL: dist, getIndexHTML, serverURL: new URL(serverURL), matrixRegistrationSecret: MATRIX_REGISTRATION_SHARED_SECRET, diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index 16219637f3..b837c5aceb 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -61,7 +61,6 @@ export function proxyAsset( ): Koa.Middleware { let filename = from.split('/').pop()!; let upstreamPath = `${assetsURL.pathname.replace(/\/$/, '')}/${filename}`; - let client = assetsURL.protocol === 'https:' ? https : http; // Direct upstream proxy. Replaces the previous koa-proxies + http-proxy // stack which forwarded `req.headers` verbatim into Node's // `http.ClientRequest`; under HTTP/2 that included pseudo-headers @@ -76,58 +75,95 @@ export function proxyAsset( if (ctxt.path !== from) { return next(); } + await pipeUpstreamAsset(ctxt, assetsURL, upstreamPath, opts); + }; +} - let forwardedHeaders: Record = {}; - for (let [name, value] of Object.entries(ctxt.req.headers)) { - if (name.startsWith(':')) continue; - // Node's http.ClientRequest rejects connection-specific hop-by-hop - // headers when targeting an HTTP/1.1 upstream. - if (name === 'host') continue; - if (typeof value === 'string') { - forwardedHeaders[name] = value; - } else if (Array.isArray(value)) { - forwardedHeaders[name] = value.join(', '); - } +// Proxy any request whose path begins with one of `prefixes` (or equals a +// prefix exactly, for single-file entries) to the host-dist upstream, +// preserving the full request path. Used in Codespace previews where the +// host app is rewritten to load its assets from the realm server's own +// origin (so the browser stays same-origin and no cross-origin CORS headers +// are needed); the realm then proxies those `/assets/…`, `/@embroider/…` +// and favicon requests to the actual host bundle (S3/CloudFront). In normal +// deployments the served HTML points assets straight at the host CDN, so +// these paths are never requested from the realm origin and this middleware +// is dormant. +export function proxyAssetPaths( + prefixes: string[], + upstreamURL: URL, + opts?: ProxyOptions, +): Koa.Middleware { + let base = upstreamURL.pathname.replace(/\/$/, ''); + return async (ctxt, next) => { + if (ctxt.method !== 'GET' && ctxt.method !== 'HEAD') { + return next(); } - for (let [key, value] of Object.entries(opts?.requestHeaders ?? {})) { - forwardedHeaders[key] = value; + let path = ctxt.path; + if (!prefixes.some((p) => path === p || path.startsWith(p))) { + return next(); } + await pipeUpstreamAsset(ctxt, upstreamURL, `${base}${path}`, opts); + }; +} - let upstreamRes = await new Promise( - (resolve, reject) => { - let upstreamReq = client.request( - { - method: ctxt.method, - hostname: assetsURL.hostname, - // `assetsURL.port` is the empty string for default-port URLs; - // fall through to the protocol default. - port: assetsURL.port || (client === https ? 443 : 80), - path: upstreamPath, - headers: forwardedHeaders, - }, - resolve, - ); - upstreamReq.on('error', reject); - upstreamReq.end(); - }, - ); - - ctxt.status = upstreamRes.statusCode ?? 502; - for (let [name, value] of Object.entries(upstreamRes.headers)) { - if (value == null) continue; - // Strip hop-by-hop headers (Node manages them per-connection) plus - // anything else the h2 response layer will reject. `host` is - // irrelevant on the response side. - if (H2_FORBIDDEN_RESPONSE_HEADERS.has(name.toLowerCase())) { - continue; - } - ctxt.set(name, Array.isArray(value) ? value.map(String) : String(value)); +async function pipeUpstreamAsset( + ctxt: Koa.Context, + assetsURL: URL, + upstreamPath: string, + opts?: ProxyOptions, +): Promise { + let client = assetsURL.protocol === 'https:' ? https : http; + let forwardedHeaders: Record = {}; + for (let [name, value] of Object.entries(ctxt.req.headers)) { + if (name.startsWith(':')) continue; + // Node's http.ClientRequest rejects connection-specific hop-by-hop + // headers when targeting an HTTP/1.1 upstream. + if (name === 'host') continue; + if (typeof value === 'string') { + forwardedHeaders[name] = value; + } else if (Array.isArray(value)) { + forwardedHeaders[name] = value.join(', '); } - for (let [key, value] of Object.entries(opts?.responseHeaders ?? {})) { - ctxt.set(key, value); + } + for (let [key, value] of Object.entries(opts?.requestHeaders ?? {})) { + forwardedHeaders[key] = value; + } + + let upstreamRes = await new Promise( + (resolve, reject) => { + let upstreamReq = client.request( + { + method: ctxt.method, + hostname: assetsURL.hostname, + // `assetsURL.port` is the empty string for default-port URLs; + // fall through to the protocol default. + port: assetsURL.port || (client === https ? 443 : 80), + path: upstreamPath, + headers: forwardedHeaders, + }, + resolve, + ); + upstreamReq.on('error', reject); + upstreamReq.end(); + }, + ); + + ctxt.status = upstreamRes.statusCode ?? 502; + for (let [name, value] of Object.entries(upstreamRes.headers)) { + if (value == null) continue; + // Strip hop-by-hop headers (Node manages them per-connection) plus + // anything else the h2 response layer will reject. `host` is + // irrelevant on the response side. + if (H2_FORBIDDEN_RESPONSE_HEADERS.has(name.toLowerCase())) { + continue; } - ctxt.body = upstreamRes; - }; + ctxt.set(name, Array.isArray(value) ? value.map(String) : String(value)); + } + for (let [key, value] of Object.entries(opts?.responseHeaders ?? {})) { + ctxt.set(key, value); + } + ctxt.body = upstreamRes; } // Add middleware to handle method override for QUERY diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 787d9164d7..2a41e88a42 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -20,6 +20,7 @@ import { ecsMetadata, methodOverrideSupport, proxyAsset, + proxyAssetPaths, } from './middleware/index.ts'; import convertAcceptHeaderQueryParam from './middleware/convert-accept-header-qp.ts'; @@ -877,6 +878,7 @@ export class RealmServer { private queue: QueuePublisher; private definitionLookup: DefinitionLookup; private assetsURL: URL; + private hostDistURL: URL; private getIndexHTML: () => Promise; private serverURL: URL; private matrixRegistrationSecret: string | undefined; @@ -912,6 +914,7 @@ export class RealmServer { queue, definitionLookup, assetsURL, + hostDistURL, getIndexHTML, matrixRegistrationSecret, matrixAdminUsername, @@ -934,6 +937,12 @@ export class RealmServer { queue: QueuePublisher; definitionLookup: DefinitionLookup; assetsURL: URL; + // Upstream for proxying host-app static assets (`/assets/…`, + // `/@embroider/…`, favicons) when the served HTML points them at the + // realm server's own origin — see `proxyAssetPaths`. Defaults to + // `assetsURL`; only differs when assets are rewritten to be same-origin + // (Codespace previews) while the actual bundle still lives elsewhere. + hostDistURL?: URL; getIndexHTML: () => Promise; matrixRegistrationSecret?: string; matrixAdminUsername?: string; @@ -976,6 +985,7 @@ export class RealmServer { this.queue = queue; this.definitionLookup = definitionLookup; this.assetsURL = assetsURL; + this.hostDistURL = hostDistURL ?? assetsURL; this.getIndexHTML = getIndexHTML; this.matrixRegistrationSecret = matrixRegistrationSecret; this.matrixAdminUsername = matrixAdminUsername; @@ -1096,12 +1106,29 @@ export class RealmServer { }), ) .use( - proxyAsset('/auth-service-worker.js', this.assetsURL, { + proxyAsset('/auth-service-worker.js', this.hostDistURL, { requestHeaders: { 'accept-encoding': 'identity', }, }), ) + // When the served HTML rewrites asset URLs to the realm server's own + // origin (Codespace previews, where assetsURL === serverURL so the + // browser stays same-origin and avoids cross-origin CORS on ES-module + // scripts), proxy those static host-app requests through to the real + // bundle. Dormant in normal deployments: there the HTML points assets + // straight at the host CDN, so these paths are never requested here. + .use( + proxyAssetPaths( + [ + '/assets/', + '/@embroider/', + '/boxel-favicon.png', + '/boxel-webclip.png', + ], + this.hostDistURL, + ), + ) .use(serveIndex) .use(serveFromRealm); From 5d6f0958af2b9046c0f05d80d3ad4901bc233e3e Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 12:33:07 -0500 Subject: [PATCH 25/37] fix(codespaces): document that forwarded ports must be set public externally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reviewer's Matrix login 401'd because port 8008 was private: a private GitHub-forwarded port auth-gates cross-origin XHR at the edge (302/401). The realm (4201), icons (4206) and Matrix (8008) are cross-origin to each other, so all three must be public. This can't be done from inside the Codespace — the ambient GITHUB_TOKEN lacks the `codespace` scope, so `gh codespace ports visibility` exits 4. (The earlier inline call never actually worked.) Make the attempt best-effort and, on failure, print clear instructions to set the ports public from the VS Code Ports panel (or via gh from outside); visibility persists per-codespace, so it's a one-time step. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 38 ++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index bbe6cd31da..6c7102b1e5 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -56,9 +56,32 @@ export PGDATABASE=boxel # block on a full index of every bootstrap realm; cards prerender on demand. export REALM_SERVER_SKIP_BOOT_INDEX=true -# ── Make forwarded ports public so the S3 host can reach this backend ── -echo "==> Making forwarded ports public..." -gh codespace ports visibility 4201:public 4206:public 8008:public -c "$CODESPACE_NAME" 2>/dev/null || true +# ── Forwarded ports must be public so the reviewer's browser can reach the +# realm (4201), the icons server (4206) and Matrix/Synapse (8008) ── +# These start cross-origin to each other, and a *private* forwarded port +# auth-gates cross-origin XHR at the GitHub edge (302 on navigation, 401 on +# fetch) — that's what 401s the browser's Matrix login. They must be public. +# +# This can't be automated from inside the Codespace: the ambient GITHUB_TOKEN +# lacks the `codespace` scope, so `gh codespace ports visibility` fails here +# (exit 4). Port visibility can only be set from OUTSIDE — either the VS Code +# "Ports" panel (right-click → Port Visibility → Public) or `gh codespace +# ports visibility 4201:public 4206:public 8008:public -c ` from a +# machine whose gh has the codespace scope. Visibility persists per-codespace +# once set, so it's a one-time step. We still attempt it (harmless, and it +# works in setups where a scoped token is present) and warn clearly if not. +echo "==> Attempting to make forwarded ports public..." +PORTS_PUBLIC_OK=1 +for p in 4201 4206 8008; do + gh codespace ports visibility "${p}:public" -c "$CODESPACE_NAME" >/dev/null 2>&1 || PORTS_PUBLIC_OK=0 +done +if [ "$PORTS_PUBLIC_OK" = 1 ]; then + echo "==> Forwarded ports set public." +else + echo "Warning: could not set forwarded ports public from inside the Codespace." + echo " Set ports 4201, 4206 and 8008 to Public in the VS Code Ports" + echo " panel (or via gh from outside), or reviewer login will 401." +fi # ── Host bundle (generic CI/S3 build; config injected at serve time) ── # No per-Codespace rebuild or target-file push: the codespaces-preview @@ -223,8 +246,13 @@ echo " ${REALM_SERVER_URL}" echo "" echo " Log in with: user / password" echo "" -echo " (Assets are served from ${BOXEL_HOST_URL};" -echo " Matrix ${MATRIX_PUBLIC_URL})" +if [ "${PORTS_PUBLIC_OK:-0}" != 1 ]; then + echo " IF LOGIN 401s: ports 4201, 4206 and 8008 must be Public. Set them" + echo " in the VS Code Ports panel (right-click → Port Visibility → Public)." + echo "" +fi +echo " (Assets are proxied through the realm origin; Matrix is at" +echo " ${MATRIX_PUBLIC_URL}; host bundle ${BOXEL_HOST_URL})" echo "============================================" # This script is launched detached from postStartCommand so the Codespace From 6a5f75271f10f949f79de530b151bf7d0331169f Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 13:04:55 -0500 Subject: [PATCH 26/37] fix(codespaces): register realm_server Matrix users so login doesn't 500 /_server-session 500'd with "Unable to login to matrix as user realm_server: 403 M_FORBIDDEN". The realm server authenticates to Synapse as realm_server (password derived from REALM_SECRET_SEED) to mint session tokens, but start-services.sh only registered the `user` test account, so realm_server never existed. Run register-realm-users with the matching REALM_SECRET_SEED before launching the realm so realm_server and the per-realm users exist with the seed-derived password. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 6c7102b1e5..c2a06a582a 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -152,6 +152,18 @@ if [ -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]; then export MATRIX_REGISTRATION_SHARED_SECRET fi +# Register the realm server's own Matrix users (realm_server plus the +# per-realm users: base_realm, catalog_realm, skills_realm, openrouter_realm, +# …). The realm server authenticates to Synapse as `realm_server` to mint +# `/_server-session` tokens, logging in with a password derived from +# REALM_SECRET_SEED (passwordFromSeed). If that user doesn't exist Synapse +# returns 403 and every login 500s at /_server-session. register-realm-users +# registers them with the matching seed-derived password, so the seed here +# MUST equal the one the realm server runs with (REALM_SECRET_SEED below). +echo "==> Registering realm Matrix users (realm_server, base/catalog/skills/openrouter)..." +(cd packages/matrix && REALM_SECRET_SEED="shhh! it's a secret" MATRIX_URL=http://localhost:8008 pnpm register-realm-users) \ + || echo "Warning: realm Matrix user registration failed; login will 500 at /_server-session." + # Seed a reviewer login (user / password) on this Codespace's own Synapse so # the preview is loginnable. This is the same dev user local development uses; # it's only as exposed as the forwarded Matrix port (see the abuse note in the From 967b015fce3e72d3e974fc0dd3ea598217fb0380 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 14:02:40 -0500 Subject: [PATCH 27/37] fix(codespaces): assert https so realm content resolves (fixes 404s) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All card/module requests 404'd while login worked. Realm content is matched by full URL via fullRequestURL/virtualNetwork, which derives the scheme from x-forwarded-proto. GitHub Codespaces port forwarding terminates TLS at the edge and forwards plain HTTP with no x-forwarded-proto, so the realm built http:// URLs that matched none of the realms registered under their https identities → 404. Path-based control endpoints (login, _server-session) were unaffected, which is why login worked but cards didn't. Add REALM_SERVER_ASSUME_HTTPS (set in the Codespace) to assert the external scheme via a front middleware. Off by default; production load balancers set x-forwarded-proto themselves. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 1 + packages/realm-server/server.ts | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index c2a06a582a..8b3077a917 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -198,6 +198,7 @@ PGPORT=5435 \ PGDATABASE=boxel \ LOG_LEVELS='*=info' \ ASSETS_URL_OVERRIDE="${REALM_SERVER_URL}" \ +REALM_SERVER_ASSUME_HTTPS=true \ REALM_SERVER_SECRET_SEED="mum's the word" \ REALM_SECRET_SEED="shhh! it's a secret" \ GRAFANA_SECRET="shhh! it's a secret" \ diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 2a41e88a42..b9cd31933d 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -1030,6 +1030,21 @@ export class RealmServer { }); let app = new Koa() + .use(async (ctx, next) => { + // When a TLS-terminating proxy forwards plain HTTP without an + // x-forwarded-proto header (GitHub Codespaces port forwarding does + // this), the realm sees every request as http and builds realm + // content URLs as http://… (see fullRequestURL). Those match none of + // the realms registered under their https identities, so every + // card/module request 404s while path-based control endpoints still + // work. REALM_SERVER_ASSUME_HTTPS asserts the real external scheme so + // URL resolution matches. Off by default; production load balancers + // set x-forwarded-proto themselves, so this never fires there. + if (process.env.REALM_SERVER_ASSUME_HTTPS === 'true') { + ctx.req.headers['x-forwarded-proto'] = 'https'; + } + await next(); + }) .use(httpLogging) .use(ecsMetadata) .use( From b38286f739900941cd2c4ae0a38f9ec84a8ac803 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 15:43:04 -0500 Subject: [PATCH 28/37] fix(codespaces): tear down prior services on (re)start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A re-run of start-services.sh would hit EADDRINUSE on 4201 because the previous realm was still bound, exit, and leave the old (pre-change) realm serving — silently masking the change under test. Kill the node service chain at the top before starting; Docker Postgres/Synapse are re-asserted idempotently. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 8b3077a917..077cb8da28 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -27,6 +27,20 @@ cd /workspaces/boxel # mise provides the pinned node/pnpm/ts-node toolchain. eval "$(mise activate bash)" +# Tear down any prior service instances before (re)starting. On a fresh +# Codespace this is a no-op, but on a re-run (after a code pull, or a manual +# restart) a still-running realm holds port 4201, so the new realm would hit +# EADDRINUSE, exit, and leave the *old* (pre-change) realm serving — silently +# masking the very change being tested. Kill the node service chain first; +# Postgres/Synapse run in Docker and are re-asserted idempotently below. +echo "==> Stopping any previously running services..." +pkill -9 -f 'ts-node.*main' 2>/dev/null || true +pkill -9 -f 'start:prerender' 2>/dev/null || true +pkill -9 -f 'start:worker' 2>/dev/null || true +pkill -9 -f 'start:icons' 2>/dev/null || true +pkill -9 -f 'prerender-manager' 2>/dev/null || true +sleep 2 + CODESPACE_NAME="${CODESPACE_NAME:?CODESPACE_NAME must be set}" FWD_DOMAIN="${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" From d080a9fa6aeed7049d50a811960248ef6262d905 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 16:16:24 -0500 Subject: [PATCH 29/37] fix(codespaces): pin Host too, not just proto, for realm URL matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The https-assert alone wasn't enough: GitHub's tunnel forwards to the backend with Host: localhost:, so the realm built https://localhost/… URLs that still matched no realm. Pin the Host to the realm's own serverURL alongside x-forwarded-proto so realm-content URL resolution matches the https:// identities. Still gated by REALM_SERVER_ASSUME_HTTPS (off in production). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/realm-server/server.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index b9cd31933d..04313bad17 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -1031,17 +1031,19 @@ export class RealmServer { let app = new Koa() .use(async (ctx, next) => { - // When a TLS-terminating proxy forwards plain HTTP without an - // x-forwarded-proto header (GitHub Codespaces port forwarding does - // this), the realm sees every request as http and builds realm - // content URLs as http://… (see fullRequestURL). Those match none of - // the realms registered under their https identities, so every - // card/module request 404s while path-based control endpoints still - // work. REALM_SERVER_ASSUME_HTTPS asserts the real external scheme so - // URL resolution matches. Off by default; production load balancers - // set x-forwarded-proto themselves, so this never fires there. + // GitHub Codespaces port forwarding terminates TLS at the edge and + // forwards to the backend over plain HTTP with `Host: localhost:` + // and no x-forwarded-proto. The realm then builds realm-content URLs + // as http://localhost/… (see fullRequestURL), which match none of the + // realms registered under their https:// identities — so + // every card/module request 404s while path-based control endpoints + // (login, _server-session, assets) still work. REALM_SERVER_ASSUME_HTTPS + // pins the scheme and host to the realm's own serverURL so URL + // resolution matches. Off by default; production load balancers set + // these headers correctly, so this never fires there. if (process.env.REALM_SERVER_ASSUME_HTTPS === 'true') { ctx.req.headers['x-forwarded-proto'] = 'https'; + ctx.req.headers['host'] = this.serverURL.host; } await next(); }) From 555e929ddc68ddd25aa12d9dd1279047f187c9d7 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 16:49:40 -0500 Subject: [PATCH 30/37] fix(codespaces): rewrite seeded realm permissions to the hosted URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bootstrap realms (catalog, skills, openrouter) 403'd: migrations seed realm_user_permissions with hardcoded https://localhost:4201/… URLs, but the realm is served under the codespace's forwarded URL, so the *:read rows matched no realm. Env mode already solves this by rewriting those rows to its Traefik hostname (fixupEnvironmentModePermissions). Generalize that to honor an explicit REALM_SERVER_PERMISSIONS_BASE_URL override and set it (to the forwarded realm URL) in the Codespace, so both hosting modes share one rewrite path. No-op in production (neither var set). Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 1 + packages/postgres/pg-adapter.ts | 68 ++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 077cb8da28..e86d6e9e1d 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -213,6 +213,7 @@ PGDATABASE=boxel \ LOG_LEVELS='*=info' \ ASSETS_URL_OVERRIDE="${REALM_SERVER_URL}" \ REALM_SERVER_ASSUME_HTTPS=true \ +REALM_SERVER_PERMISSIONS_BASE_URL="${REALM_SERVER_URL}" \ REALM_SERVER_SECRET_SEED="mum's the word" \ REALM_SECRET_SEED="shhh! it's a secret" \ GRAFANA_SECRET="shhh! it's a secret" \ diff --git a/packages/postgres/pg-adapter.ts b/packages/postgres/pg-adapter.ts index 07ca49b40f..d054966694 100644 --- a/packages/postgres/pg-adapter.ts +++ b/packages/postgres/pg-adapter.ts @@ -662,7 +662,7 @@ export class PgAdapter implements DBAdapter { ignorePattern: '.*\\.eslintrc\\.js', log: enableLogging ? (...args) => log.info(...args) : () => undefined, }); - await this.fixupEnvironmentModePermissions(config); + await this.fixupHostedRealmPermissions(config); return; } catch (err: any) { if (!err.message?.includes('Another migration is already running')) { @@ -674,31 +674,45 @@ export class PgAdapter implements DBAdapter { } } - // In environment mode, migrations seed realm_user_permissions with hardcoded - // localhost:4201/4202 URLs. Rewrite them to the Traefik hostnames so realm - // ownership lookups work. - private async fixupEnvironmentModePermissions(config: Config) { + // Migrations seed realm_user_permissions with hardcoded localhost:4201/4202 + // URLs. When the realm server is hosted under a different public origin, its + // realm-ownership and read-permission lookups key off the realm's actual + // URL, so those rows must be rewritten to that origin — otherwise every + // bootstrap realm 403s. Two hosting modes need this: + // - environment mode (BOXEL_ENVIRONMENT): rewrite to the Traefik hostnames + // derived from the env slug (realm-server / realm-test). + // - an explicit public-URL override (REALM_SERVER_PERMISSIONS_BASE_URL): + // single-origin deployments such as GitHub Codespaces, where the realm + // is served under one forwarded https URL with no separate :4202 peer. + // The override wins when both are set. + private async fixupHostedRealmPermissions(config: Config) { + let baseOverride = process.env.REALM_SERVER_PERMISSIONS_BASE_URL; let branch = process.env.BOXEL_ENVIRONMENT; - if (!branch) { + + let realmServerUrl: string | undefined; + let realmTestUrl: string | undefined; + if (baseOverride) { + realmServerUrl = baseOverride.replace(/\/$/, ''); + } else if (branch) { + let slug = branch + .toLowerCase() + .replace(/\//g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); + realmServerUrl = `http://realm-server.${slug}.localhost`; + realmTestUrl = `http://realm-test.${slug}.localhost`; + } else { return; } - let slug = branch - .toLowerCase() - .replace(/\//g, '-') - .replace(/[^a-z0-9-]/g, '') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); let client = new Client(config); try { await client.connect(); - let realmServerUrl = `http://realm-server.${slug}.localhost`; - let realmTestUrl = `http://realm-test.${slug}.localhost`; // Match both http and https canonicals — realm-server speaks HTTPS in // local dev now, so a DB seeded after the CS-11114 flip stores // `https://localhost:42XX/...` permission rows; older rows can still - // be on `http://`. The regex collapses both into the env-mode - // Traefik hostname. + // be on `http://`. The regex collapses both into the hosted origin. let result = await client.query( `UPDATE realm_user_permissions SET realm_url = regexp_replace(realm_url, '^https?://localhost:4201/', $1) @@ -707,19 +721,21 @@ export class PgAdapter implements DBAdapter { ); if (result.rowCount && result.rowCount > 0) { log.info( - `Environment mode: rewrote ${result.rowCount} permission URL(s) from localhost:4201 to ${realmServerUrl}`, + `Rewrote ${result.rowCount} permission URL(s) from localhost:4201 to ${realmServerUrl}`, ); } - let result2 = await client.query( - `UPDATE realm_user_permissions - SET realm_url = regexp_replace(realm_url, '^https?://localhost:4202/', $1) - WHERE realm_url ~ '^https?://localhost:4202/'`, - [`${realmTestUrl}/`], - ); - if (result2.rowCount && result2.rowCount > 0) { - log.info( - `Environment mode: rewrote ${result2.rowCount} permission URL(s) from localhost:4202 to ${realmTestUrl}`, + if (realmTestUrl) { + let result2 = await client.query( + `UPDATE realm_user_permissions + SET realm_url = regexp_replace(realm_url, '^https?://localhost:4202/', $1) + WHERE realm_url ~ '^https?://localhost:4202/'`, + [`${realmTestUrl}/`], ); + if (result2.rowCount && result2.rowCount > 0) { + log.info( + `Rewrote ${result2.rowCount} permission URL(s) from localhost:4202 to ${realmTestUrl}`, + ); + } } } finally { await client.end(); From b281ebd92139d78b2e5c0c2bb681dc5b32e8e7f4 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 18 Jun 2026 17:04:37 -0500 Subject: [PATCH 31/37] fix(codespaces): index bootstrap realms so card instances resolve With REALM_SERVER_SKIP_BOOT_INDEX the bootstrap realms mounted but never indexed, so card instances (the AI system card, skills, catalog cards) 404'd from getCard even though their source files exist. Drop the skip so base/catalog/skills/openrouter full-index on boot. Also fix the readiness probe: _readiness-check is per-realm, so probe /base/_readiness-check (the server root 404s); ASSUME_HTTPS rewrites the localhost probe to the realm URL. Longer timeout covers the from-scratch index. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index e86d6e9e1d..46b88f2522 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -13,13 +13,13 @@ # hand (not `mise run dev`) only so its realm toUrls can be the public # Codespace URLs; it still picks up the HTTPS cert via env-vars.sh. # -# A host app (vite) IS run locally because the realm server hard-requires a -# reachable host: main.ts fetches HOST_URL at startup and process.exit(-2)s -# if it can't (and the prerenderer renders cards against it). This local host -# is distinct from the reviewer-facing S3 build; it exists so the realm server -# boots and cards can prerender. The realm starts in mount-and-serve mode -# (REALM_SERVER_SKIP_BOOT_INDEX) so readiness doesn't block on a full -# from-scratch index; cards prerender on demand instead. +# The realm server hard-requires a reachable host app: main.ts fetches +# HOST_URL at startup and process.exit(-2)s if it can't, and the prerenderer +# renders cards against it. Rather than build a second host here, HOST_URL +# points at the CI/S3 preview bundle (see below). The bootstrap realms (base, +# catalog, skills, openrouter) full-index on boot so their card instances are +# queryable — the AI system card, skills and catalog browsing all read from +# that index, so readiness waits for the index to finish. set -euo pipefail cd /workspaces/boxel @@ -66,10 +66,6 @@ export ICONS_URL="http://localhost:4206" export PGPORT=5435 export PGDATABASE=boxel -# Mount-and-serve: skip the from-scratch boot index so readiness doesn't -# block on a full index of every bootstrap realm; cards prerender on demand. -export REALM_SERVER_SKIP_BOOT_INDEX=true - # ── Forwarded ports must be public so the reviewer's browser can reach the # realm (4201), the icons server (4206) and Matrix/Synapse (8008) ── # These start cross-origin to each other, and a *private* forwarded port @@ -224,7 +220,6 @@ RESOLVED_MATRIX_URL="${MATRIX_PUBLIC_URL}" \ MATRIX_SERVER_NAME=localhost \ REALM_SERVER_MATRIX_USERNAME=realm_server \ ENABLE_FILE_WATCHER=true \ -REALM_SERVER_SKIP_BOOT_INDEX=true \ pnpm --dir=packages/realm-server exec ts-node \ --transpileOnly main \ --port=4201 \ @@ -257,11 +252,16 @@ REALM_SERVER_SKIP_BOOT_INDEX=true \ REALM_PID=$! # ── Wait for realm server readiness ── -# The realm server serves plain HTTP on 4201 (GitHub's edge terminates TLS). -echo "==> Waiting for realm server to be ready..." -timeout 300 bash -c \ - 'until curl -sf "http://localhost:4201/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 2; done' \ - || echo "Warning: realm server readiness check timed out after 5 minutes" +# _readiness-check is a per-realm endpoint that reports ready once the realm +# has finished indexing, so probe the base realm specifically (not the server +# root, which 404s). The realm serves plain HTTP on 4201 and REALM_SERVER_ASSUME_HTTPS +# rewrites this localhost probe's host to the public realm URL so it resolves +# to the mounted realm. The long timeout covers the bootstrap from-scratch +# index (base is large); the realm still serves modules while it indexes. +echo "==> Waiting for realm server (and bootstrap index) to be ready..." +timeout 900 bash -c \ + 'until curl -sf "http://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" >/dev/null 2>&1; do sleep 3; done' \ + || echo "Warning: realm server readiness check timed out" echo "" echo "============================================" From 8e75da61ae266187d84545a1e9ccba50294cf281 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 19 Jun 2026 08:50:13 -0500 Subject: [PATCH 32/37] fix(codespaces): point worker/prerender REALM_BASE_URL at the forwarded URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexing failed on every realm because the worker/prerender mise tasks addressed realms at https://localhost:4201 (the standard-mode default) while the realm is mounted at the codespace's forwarded URL and serves plain HTTP — so base index hit a TLS error and the rest got the host SPA's HTML instead of JSON. mise's _.source re-evaluation ignores an ambient REALM_BASE_URL override, but CODESPACE_NAME does reach it, so set REALM_BASE_URL from it here — mirroring the env-mode branch's single consistent realm URL, with the GitHub edge in place of Traefik. Co-Authored-By: Claude Opus 4.8 (1M context) --- mise-tasks/lib/env-vars.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index 4c3c111d23..152eec294e 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -219,6 +219,22 @@ else fi unset _BOXEL_DEV_CERT_DIR _BOXEL_DEV_CERT_FILE _BOXEL_DEV_KEY_FILE + # GitHub Codespaces (outside env mode): the realm server is reached at the + # forwarded edge URL (https://-4201.), not localhost — GitHub + # terminates TLS at the edge while the realm serves plain HTTP and asserts + # the external scheme via REALM_SERVER_ASSUME_HTTPS. The worker and prerender + # address realms by REALM_BASE_URL, so it must be that forwarded URL or every + # from-scratch index fails (the default https://localhost:4201 yields a TLS + # error against the plain-HTTP realm for base, and the host SPA's HTML + # instead of realm JSON for the rest). Mirrors the env-mode branch's single + # REALM_BASE_URL, with the forwarded edge in place of Traefik; internal + # services (matrix, worker manager, prerender, icons) stay on localhost. + # CODESPACE_NAME is set by the Codespaces platform, so this never fires in + # local dev or CI. + if [ -n "${CODESPACE_NAME:-}" ]; then + export REALM_BASE_URL="https://${CODESPACE_NAME}-4201.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" + fi + # Puppeteer 24.35 (and the lockfile's tree) bundles Chrome 143, which # has a known h2 stream-window bug that hangs the dev prerender forever # on the first cold-start fetch of vite's large pre-optimized From 348348c467b704c2bb57247f8bb40454ab8e7236 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 19 Jun 2026 09:23:01 -0500 Subject: [PATCH 33/37] fix(codespaces): teardown must match real ts-node process names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The teardown grepped `prerender-manager` (matches nothing — the process is `prerender/manager-server`) and the npm-script wrappers rather than the ts-node entry points, so stale prerender managers/workers survived a re-run. A leftover manager then held :4222 and the new one EADDRINUSE'd and died, so the worker's render dispatch failed and indexing stalled. Match the actual entry points (transpileOnly main/worker, manager-server, prerender-server) plus the wrappers. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 46b88f2522..f70d2797c0 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -34,11 +34,27 @@ eval "$(mise activate bash)" # masking the very change being tested. Kill the node service chain first; # Postgres/Synapse run in Docker and are re-asserted idempotently below. echo "==> Stopping any previously running services..." -pkill -9 -f 'ts-node.*main' 2>/dev/null || true -pkill -9 -f 'start:prerender' 2>/dev/null || true -pkill -9 -f 'start:worker' 2>/dev/null || true -pkill -9 -f 'start:icons' 2>/dev/null || true -pkill -9 -f 'prerender-manager' 2>/dev/null || true +# Match the actual ts-node entry-point processes (e.g. the realm is +# `ts-node … --transpileOnly main`, the prerender manager is `ts-node … +# prerender/manager-server`), not just the npm-script wrappers. Killing only +# the wrappers (or a wrong pattern like `prerender-manager`, which matches +# nothing — the process is `manager-server`) leaves the real service holding +# its port, so the new one EADDRINUSEs and dies, silently masking the change +# under test. `transpileOnly worker` matches both the worker and the +# worker-manager. Postgres/Synapse run in Docker and are re-asserted below. +for _pat in \ + 'transpileOnly main' \ + 'transpileOnly worker' \ + 'manager-server' \ + 'prerender-server' \ + 'services:prerender' \ + 'services:worker' \ + 'start:prerender' \ + 'start:worker' \ + 'start:icons' \ + 'start-icons'; do + pkill -9 -f "$_pat" 2>/dev/null || true +done sleep 2 CODESPACE_NAME="${CODESPACE_NAME:?CODESPACE_NAME must be set}" From 8ddebadfa0ce7d0d44357bbc9a553c64ba5c4f83 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 19 Jun 2026 11:04:36 -0500 Subject: [PATCH 34/37] feat(codespaces): proxy Matrix + icons through the realm (single origin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So a reviewer's browser only talks to the one forwarded realm port instead of separate cross-origin Matrix (8008) and icons (4206) ports — which would each need to be made public. Adds proxyRequest (a method+body+streaming reverse proxy), routes /_matrix, /.well-known/matrix and /@cardstack/boxel-icons to their localhost backends, and points the host's matrixURL/iconsURL at the realm origin. Gated by REALM_SERVER_PROXY_MATRIX_ICONS; dormant in normal deployments. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/start-services.sh | 3 +- packages/realm-server/handlers/serve-index.ts | 24 +++++-- packages/realm-server/middleware/index.ts | 64 +++++++++++++++++++ packages/realm-server/server.ts | 37 +++++++++++ 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index f70d2797c0..ab88f423a8 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -232,10 +232,11 @@ GRAFANA_SECRET="shhh! it's a secret" \ LOW_CREDIT_THRESHOLD="${LOW_CREDIT_THRESHOLD:-2000}" \ HOST_URL="$BOXEL_HOST_URL" \ MATRIX_URL=http://localhost:8008 \ -RESOLVED_MATRIX_URL="${MATRIX_PUBLIC_URL}" \ MATRIX_SERVER_NAME=localhost \ REALM_SERVER_MATRIX_USERNAME=realm_server \ ENABLE_FILE_WATCHER=true \ +REALM_SERVER_PROXY_MATRIX_ICONS=true \ +ICONS_BACKEND_URL=http://localhost:4206 \ pnpm --dir=packages/realm-server exec ts-node \ --transpileOnly main \ --port=4201 \ diff --git a/packages/realm-server/handlers/serve-index.ts b/packages/realm-server/handlers/serve-index.ts index c68fb7caa1..78a4fbc6e2 100644 --- a/packages/realm-server/handlers/serve-index.ts +++ b/packages/realm-server/handlers/serve-index.ts @@ -119,20 +119,32 @@ export function createServeIndex(deps: ServeIndexDeps): ServeIndexHandlers { config.publishedRealmBoxelSiteDomain = serverURL.host; } + // Codespace single-origin: when the realm server reverse-proxies + // Matrix and the icons server onto its own origin (see + // proxyRequest in server.ts), point the host at this origin for + // both, so the browser only ever talks to the one forwarded port. + let proxyMatrixIcons = + process.env.REALM_SERVER_PROXY_MATRIX_ICONS === 'true'; + config = merge({}, config, { hostsOwnAssets: false, assetsURL: assetsURL.href, // The browser-facing Matrix URL can differ from the realm // server's own backend connection (e.g. in a Codespace the // backend talks to Synapse on localhost while the browser must - // use the public forwarded URL). RESOLVED_MATRIX_URL lets a - // deployment inject the browser-facing URL without rerouting the - // realm's own Matrix client. Falls back to the backend URL. - matrixURL: ( - process.env.RESOLVED_MATRIX_URL ?? matrixClient.matrixURL.href - ).replace(/\/$/, ''), + // use the public forwarded URL). When proxying Matrix through + // this origin, point the browser here; otherwise RESOLVED_MATRIX_URL + // (or the backend URL) is the browser-facing Matrix URL. + matrixURL: proxyMatrixIcons + ? serverURL.origin + : ( + process.env.RESOLVED_MATRIX_URL ?? matrixClient.matrixURL.href + ).replace(/\/$/, ''), matrixServerName: process.env.MATRIX_SERVER_NAME || matrixClient.matrixURL.hostname, + // Icons are fetched as `${iconsURL}/@cardstack/boxel-icons/...`; + // when proxied, serve them from this origin too. + ...(proxyMatrixIcons ? { iconsURL: serverURL.origin } : {}), realmServerURL: serverURL.href, resolvedBaseRealmURL: rewriteRealmURL(config.resolvedBaseRealmURL), resolvedCatalogRealmURL: rewriteRealmURL( diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index b837c5aceb..3a4ffd2d48 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -166,6 +166,70 @@ async function pipeUpstreamAsset( ctxt.body = upstreamRes; } +// Reverse-proxy any request whose path begins with one of `prefixes` to a +// backend, preserving method, path+query, request body, and streaming the +// response back. Unlike proxyAsset/proxyAssetPaths (GET-only), this forwards +// the request body, so it handles POST/PUT and long-poll responses (Matrix +// sync). Used in Codespace previews to fold Matrix (`/_matrix`, +// `/.well-known/matrix`) and the icons server (`/@cardstack/boxel-icons`) +// onto the realm server's own origin, so the browser only ever talks to the +// one forwarded port — no separate cross-origin ports to make public. Must +// run before any body-consuming middleware so `ctxt.req` is still unread. +// Dormant in normal deployments (those paths aren't routed here). +export function proxyRequest( + prefixes: string[], + upstreamURL: URL, +): Koa.Middleware { + let client = upstreamURL.protocol === 'https:' ? https : http; + let base = upstreamURL.pathname.replace(/\/$/, ''); + return async (ctxt, next) => { + let path = ctxt.path; + if (!prefixes.some((p) => path === p || path.startsWith(p))) { + return next(); + } + + let forwardedHeaders: Record = {}; + for (let [name, value] of Object.entries(ctxt.req.headers)) { + if (name.startsWith(':')) continue; + if (name === 'host') continue; + if (typeof value === 'string') { + forwardedHeaders[name] = value; + } else if (Array.isArray(value)) { + forwardedHeaders[name] = value.join(', '); + } + } + + let upstreamRes = await new Promise( + (resolve, reject) => { + let upstreamReq = client.request( + { + method: ctxt.method, + hostname: upstreamURL.hostname, + port: upstreamURL.port || (client === https ? 443 : 80), + path: `${base}${ctxt.originalUrl}`, + headers: forwardedHeaders, + }, + resolve, + ); + upstreamReq.on('error', reject); + // Forward the request body for POST/PUT/etc.; for bodyless methods the + // piped request stream simply ends right away. + ctxt.req.pipe(upstreamReq); + }, + ); + + ctxt.status = upstreamRes.statusCode ?? 502; + for (let [name, value] of Object.entries(upstreamRes.headers)) { + if (value == null) continue; + if (H2_FORBIDDEN_RESPONSE_HEADERS.has(name.toLowerCase())) { + continue; + } + ctxt.set(name, Array.isArray(value) ? value.map(String) : String(value)); + } + ctxt.body = upstreamRes; + }; +} + // Add middleware to handle method override for QUERY export function methodOverrideSupport(ctxt: Koa.Context, next: Koa.Next) { const methodOverride = ctxt.request.headers['x-http-method-override']; diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 04313bad17..528cd11bc1 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -21,6 +21,7 @@ import { methodOverrideSupport, proxyAsset, proxyAssetPaths, + proxyRequest, } from './middleware/index.ts'; import convertAcceptHeaderQueryParam from './middleware/convert-accept-header-qp.ts'; @@ -1074,6 +1075,42 @@ export class RealmServer { maxAge: 86400, }), ) + // Codespace single-origin: fold Matrix and the icons server onto this + // realm server's own origin so the reviewer's browser only ever talks + // to the one forwarded port (no separate cross-origin ports to make + // public). serve-index points the host's matrixURL/iconsURL here; these + // proxies forward to the real backends. Placed before the body-reading + // middleware so the request body (Matrix POSTs) is still intact, and + // before the realm router so `/_matrix`/`/@cardstack/boxel-icons` never + // fall through to realm routing. Off in normal deployments. + .use( + (() => { + if (process.env.REALM_SERVER_PROXY_MATRIX_ICONS !== 'true') { + return async (_ctx: Koa.Context, next: Koa.Next) => next(); + } + let matrixProxy = proxyRequest( + ['/_matrix/', '/.well-known/matrix/'], + new URL(this.matrixClient.matrixURL.href), + ); + let iconsProxy = proxyRequest( + ['/@cardstack/boxel-icons/'], + new URL(process.env.ICONS_BACKEND_URL ?? 'http://localhost:4206'), + ); + return async (ctx: Koa.Context, next: Koa.Next) => { + let p = ctx.path; + if ( + p.startsWith('/_matrix/') || + p.startsWith('/.well-known/matrix/') + ) { + return matrixProxy(ctx, next); + } + if (p.startsWith('/@cardstack/boxel-icons/')) { + return iconsProxy(ctx, next); + } + return next(); + }; + })(), + ) .use(async (ctx, next) => { // Disable browser cache for all data requests to the realm server. The condition captures our supported mime types but not others, // such as assets, which we probably want to cache. From d403e8c39bfd166c1b6cc9b3afe7f93fee6f5aa8 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 19 Jun 2026 16:39:30 -0500 Subject: [PATCH 35/37] feat(codespaces): local TLS shim so indexing needs no public port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker and prerender address the realm at its forwarded https URL; they were looping back out through the GitHub edge, which needs the port public. Add a local TLS shim (env-mode's Traefik, rebuilt): /etc/hosts maps the forwarded realm hostname to 127.0.0.1 and a :443 proxy forwards to the plain-HTTP realm on :4201, so the worker/prerender reach it over loopback — no edge, no public port. Node (NODE_TLS_REJECT_UNAUTHORIZED) and Chrome (PUPPETEER_CHROME_ARGS) skip validation of the loopback self-signed cert. The browser is external and still uses the edge. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/local-tls-proxy.mjs | 65 +++++++++++++++++++++++++++++++ .devcontainer/start-services.sh | 34 ++++++++++++++++ mise-tasks/lib/env-vars.sh | 10 +++++ 3 files changed, 109 insertions(+) create mode 100644 .devcontainer/local-tls-proxy.mjs diff --git a/.devcontainer/local-tls-proxy.mjs b/.devcontainer/local-tls-proxy.mjs new file mode 100644 index 0000000000..823c377564 --- /dev/null +++ b/.devcontainer/local-tls-proxy.mjs @@ -0,0 +1,65 @@ +// Local TLS shim for GitHub Codespaces. +// +// The realm server serves plain HTTP on :4201 and GitHub's edge terminates +// TLS for the *browser*. But in-codespace clients — the index worker and the +// prerender's headless Chrome — address the realm at its canonical +// https://-4201. URL. Without help they'd reach it back out +// through the GitHub edge, which requires the port to be public (a manual +// step that can't be set from inside the codespace). +// +// Paired with an /etc/hosts entry mapping that hostname to 127.0.0.1, this +// shim terminates TLS on :443 and forwards to the realm on 127.0.0.1:4201, so +// those clients reach the realm entirely over loopback — no edge round-trip, +// no public port. It's the local equivalent of env-mode's Traefik. The +// browser is unaffected: it's external, uses real DNS, and still reaches the +// realm through GitHub's edge. +// +// Self-signed cert; in-codespace clients are configured to skip validation +// (worker: NODE_TLS_REJECT_UNAUTHORIZED=0; Chrome: --ignore-certificate-errors +// via PUPPETEER_CHROME_ARGS), since the connection is loopback-only. +import https from 'node:https'; +import http from 'node:http'; +import { readFileSync } from 'node:fs'; + +const PORT = Number(process.env.SHIM_PORT || 443); +const TARGET_PORT = Number(process.env.SHIM_TARGET_PORT || 4201); +const TARGET_HOST = process.env.SHIM_TARGET_HOST || '127.0.0.1'; + +const server = https.createServer( + { + cert: readFileSync(process.env.SHIM_CERT), + key: readFileSync(process.env.SHIM_KEY), + }, + (req, res) => { + // Forward method, path+query, headers and body verbatim; the realm's + // REALM_SERVER_ASSUME_HTTPS handling fixes up scheme/host from here. + let upstream = http.request( + { + host: TARGET_HOST, + port: TARGET_PORT, + method: req.method, + path: req.url, + headers: req.headers, + }, + (upRes) => { + res.writeHead(upRes.statusCode || 502, upRes.headers); + upRes.pipe(res); + }, + ); + upstream.on('error', (err) => { + if (!res.headersSent) res.writeHead(502); + res.end(`local-tls-shim upstream error: ${err.message}`); + }); + req.pipe(upstream); + }, +); + +server.on('error', (err) => { + console.error(`[local-tls-shim] fatal: ${err.message}`); + process.exit(1); +}); +server.listen(PORT, () => { + console.log( + `[local-tls-shim] listening :${PORT} -> ${TARGET_HOST}:${TARGET_PORT}`, + ); +}); diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index ab88f423a8..53eafb161a 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -55,6 +55,8 @@ for _pat in \ 'start-icons'; do pkill -9 -f "$_pat" 2>/dev/null || true done +# The local TLS shim binds privileged :443 and runs as root, so kill it with sudo. +sudo pkill -9 -f 'local-tls-proxy.mjs' 2>/dev/null || true sleep 2 CODESPACE_NAME="${CODESPACE_NAME:?CODESPACE_NAME must be set}" @@ -67,6 +69,38 @@ export REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" export MATRIX_PUBLIC_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" export ICONS_PUBLIC_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" +# ── Local TLS shim (env-mode's Traefik, rebuilt locally) ── +# In-codespace clients (the index worker, the prerender's headless Chrome) +# address the realm at its canonical https forwarded URL. Without this they'd +# loop back out through the GitHub edge, which requires the port to be public +# (a manual step we can't set from inside). Instead: map the forwarded realm +# hostname to 127.0.0.1 in /etc/hosts and run a TLS proxy on :443 that +# forwards to the plain-HTTP realm on :4201, so those clients reach the realm +# entirely over loopback — no edge, no public port. The browser is external +# and still reaches the realm through the edge, unaffected. +REALM_HOSTNAME="${CODESPACE_NAME}-4201.${FWD_DOMAIN}" +SHIM_DIR="$HOME/.local/share/boxel/codespace-tls" +mkdir -p "$SHIM_DIR" +if [ ! -f "$SHIM_DIR/cert.pem" ]; then + echo "==> Generating self-signed cert for the local TLS shim..." + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$SHIM_DIR/key.pem" -out "$SHIM_DIR/cert.pem" -days 3650 \ + -subj "/CN=${REALM_HOSTNAME}" \ + -addext "subjectAltName=DNS:${REALM_HOSTNAME}" 2>/dev/null \ + || echo "Warning: cert generation failed; the TLS shim won't start." +fi +if ! grep -qF "$REALM_HOSTNAME" /etc/hosts 2>/dev/null; then + echo "==> Mapping ${REALM_HOSTNAME} -> 127.0.0.1 in /etc/hosts..." + echo "127.0.0.1 ${REALM_HOSTNAME}" | sudo tee -a /etc/hosts >/dev/null \ + || echo "Warning: could not edit /etc/hosts; in-codespace clients will use the edge." +fi +echo "==> Starting local TLS shim (:443 -> 127.0.0.1:4201)..." +sudo env \ + SHIM_CERT="$SHIM_DIR/cert.pem" SHIM_KEY="$SHIM_DIR/key.pem" \ + SHIM_PORT=443 SHIM_TARGET_PORT=4201 \ + "$(command -v node)" /workspaces/boxel/.devcontainer/local-tls-proxy.mjs \ + > /tmp/local-tls-shim.log 2>&1 & + # Internal wiring runs over plain HTTP on the standard ports. No dev cert is # generated (see setup.sh): the realm server serves plain HTTP and GitHub's # port forwarding terminates TLS at its edge. Override the env-vars.sh diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index 152eec294e..3b0b6d9429 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -233,6 +233,16 @@ else # local dev or CI. if [ -n "${CODESPACE_NAME:-}" ]; then export REALM_BASE_URL="https://${CODESPACE_NAME}-4201.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" + # The worker/prerender reach that URL over the local TLS shim (see + # .devcontainer/local-tls-proxy.mjs), which presents a self-signed cert on + # a loopback connection. Tell Node (the worker) and Chrome (the prerender) + # to skip validation for it. browser-manager.ts only auto-adds the + # ignore-cert flags for https-loopback REALM_BASE_URLs, which this is not, + # so pass them explicitly via PUPPETEER_CHROME_ARGS (appended in + # browser-manager.ts). Safe here: the connection is loopback-only and this + # branch only fires inside a Codespace. + export NODE_TLS_REJECT_UNAUTHORIZED=0 + export PUPPETEER_CHROME_ARGS="--ignore-certificate-errors --allow-insecure-localhost" fi # Puppeteer 24.35 (and the lockfile's tree) bundles Chrome 143, which From f95875865df612dfe4beb436f5c9e55893a203e3 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 19 Jun 2026 17:37:43 -0500 Subject: [PATCH 36/37] Revert "feat(codespaces): local TLS shim so indexing needs no public port" This reverts commit d403e8c39bfd166c1b6cc9b3afe7f93fee6f5aa8. --- .devcontainer/local-tls-proxy.mjs | 65 ------------------------------- .devcontainer/start-services.sh | 34 ---------------- mise-tasks/lib/env-vars.sh | 10 ----- 3 files changed, 109 deletions(-) delete mode 100644 .devcontainer/local-tls-proxy.mjs diff --git a/.devcontainer/local-tls-proxy.mjs b/.devcontainer/local-tls-proxy.mjs deleted file mode 100644 index 823c377564..0000000000 --- a/.devcontainer/local-tls-proxy.mjs +++ /dev/null @@ -1,65 +0,0 @@ -// Local TLS shim for GitHub Codespaces. -// -// The realm server serves plain HTTP on :4201 and GitHub's edge terminates -// TLS for the *browser*. But in-codespace clients — the index worker and the -// prerender's headless Chrome — address the realm at its canonical -// https://-4201. URL. Without help they'd reach it back out -// through the GitHub edge, which requires the port to be public (a manual -// step that can't be set from inside the codespace). -// -// Paired with an /etc/hosts entry mapping that hostname to 127.0.0.1, this -// shim terminates TLS on :443 and forwards to the realm on 127.0.0.1:4201, so -// those clients reach the realm entirely over loopback — no edge round-trip, -// no public port. It's the local equivalent of env-mode's Traefik. The -// browser is unaffected: it's external, uses real DNS, and still reaches the -// realm through GitHub's edge. -// -// Self-signed cert; in-codespace clients are configured to skip validation -// (worker: NODE_TLS_REJECT_UNAUTHORIZED=0; Chrome: --ignore-certificate-errors -// via PUPPETEER_CHROME_ARGS), since the connection is loopback-only. -import https from 'node:https'; -import http from 'node:http'; -import { readFileSync } from 'node:fs'; - -const PORT = Number(process.env.SHIM_PORT || 443); -const TARGET_PORT = Number(process.env.SHIM_TARGET_PORT || 4201); -const TARGET_HOST = process.env.SHIM_TARGET_HOST || '127.0.0.1'; - -const server = https.createServer( - { - cert: readFileSync(process.env.SHIM_CERT), - key: readFileSync(process.env.SHIM_KEY), - }, - (req, res) => { - // Forward method, path+query, headers and body verbatim; the realm's - // REALM_SERVER_ASSUME_HTTPS handling fixes up scheme/host from here. - let upstream = http.request( - { - host: TARGET_HOST, - port: TARGET_PORT, - method: req.method, - path: req.url, - headers: req.headers, - }, - (upRes) => { - res.writeHead(upRes.statusCode || 502, upRes.headers); - upRes.pipe(res); - }, - ); - upstream.on('error', (err) => { - if (!res.headersSent) res.writeHead(502); - res.end(`local-tls-shim upstream error: ${err.message}`); - }); - req.pipe(upstream); - }, -); - -server.on('error', (err) => { - console.error(`[local-tls-shim] fatal: ${err.message}`); - process.exit(1); -}); -server.listen(PORT, () => { - console.log( - `[local-tls-shim] listening :${PORT} -> ${TARGET_HOST}:${TARGET_PORT}`, - ); -}); diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index 53eafb161a..ab88f423a8 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -55,8 +55,6 @@ for _pat in \ 'start-icons'; do pkill -9 -f "$_pat" 2>/dev/null || true done -# The local TLS shim binds privileged :443 and runs as root, so kill it with sudo. -sudo pkill -9 -f 'local-tls-proxy.mjs' 2>/dev/null || true sleep 2 CODESPACE_NAME="${CODESPACE_NAME:?CODESPACE_NAME must be set}" @@ -69,38 +67,6 @@ export REALM_SERVER_URL="https://${CODESPACE_NAME}-4201.${FWD_DOMAIN}" export MATRIX_PUBLIC_URL="https://${CODESPACE_NAME}-8008.${FWD_DOMAIN}" export ICONS_PUBLIC_URL="https://${CODESPACE_NAME}-4206.${FWD_DOMAIN}" -# ── Local TLS shim (env-mode's Traefik, rebuilt locally) ── -# In-codespace clients (the index worker, the prerender's headless Chrome) -# address the realm at its canonical https forwarded URL. Without this they'd -# loop back out through the GitHub edge, which requires the port to be public -# (a manual step we can't set from inside). Instead: map the forwarded realm -# hostname to 127.0.0.1 in /etc/hosts and run a TLS proxy on :443 that -# forwards to the plain-HTTP realm on :4201, so those clients reach the realm -# entirely over loopback — no edge, no public port. The browser is external -# and still reaches the realm through the edge, unaffected. -REALM_HOSTNAME="${CODESPACE_NAME}-4201.${FWD_DOMAIN}" -SHIM_DIR="$HOME/.local/share/boxel/codespace-tls" -mkdir -p "$SHIM_DIR" -if [ ! -f "$SHIM_DIR/cert.pem" ]; then - echo "==> Generating self-signed cert for the local TLS shim..." - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$SHIM_DIR/key.pem" -out "$SHIM_DIR/cert.pem" -days 3650 \ - -subj "/CN=${REALM_HOSTNAME}" \ - -addext "subjectAltName=DNS:${REALM_HOSTNAME}" 2>/dev/null \ - || echo "Warning: cert generation failed; the TLS shim won't start." -fi -if ! grep -qF "$REALM_HOSTNAME" /etc/hosts 2>/dev/null; then - echo "==> Mapping ${REALM_HOSTNAME} -> 127.0.0.1 in /etc/hosts..." - echo "127.0.0.1 ${REALM_HOSTNAME}" | sudo tee -a /etc/hosts >/dev/null \ - || echo "Warning: could not edit /etc/hosts; in-codespace clients will use the edge." -fi -echo "==> Starting local TLS shim (:443 -> 127.0.0.1:4201)..." -sudo env \ - SHIM_CERT="$SHIM_DIR/cert.pem" SHIM_KEY="$SHIM_DIR/key.pem" \ - SHIM_PORT=443 SHIM_TARGET_PORT=4201 \ - "$(command -v node)" /workspaces/boxel/.devcontainer/local-tls-proxy.mjs \ - > /tmp/local-tls-shim.log 2>&1 & - # Internal wiring runs over plain HTTP on the standard ports. No dev cert is # generated (see setup.sh): the realm server serves plain HTTP and GitHub's # port forwarding terminates TLS at its edge. Override the env-vars.sh diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index 3b0b6d9429..152eec294e 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -233,16 +233,6 @@ else # local dev or CI. if [ -n "${CODESPACE_NAME:-}" ]; then export REALM_BASE_URL="https://${CODESPACE_NAME}-4201.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-app.github.dev}" - # The worker/prerender reach that URL over the local TLS shim (see - # .devcontainer/local-tls-proxy.mjs), which presents a self-signed cert on - # a loopback connection. Tell Node (the worker) and Chrome (the prerender) - # to skip validation for it. browser-manager.ts only auto-adds the - # ignore-cert flags for https-loopback REALM_BASE_URLs, which this is not, - # so pass them explicitly via PUPPETEER_CHROME_ARGS (appended in - # browser-manager.ts). Safe here: the connection is loopback-only and this - # branch only fires inside a Codespace. - export NODE_TLS_REJECT_UNAUTHORIZED=0 - export PUPPETEER_CHROME_ARGS="--ignore-certificate-errors --allow-insecure-localhost" fi # Puppeteer 24.35 (and the lockfile's tree) bundles Chrome 143, which From d379c80432c91612c3f4865d4ecbd24c76217f80 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 19 Jun 2026 21:36:16 -0500 Subject: [PATCH 37/37] fix(codespaces): wait for Docker so postStart auto-start is reliable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit postStartCommand fires on every container start, racing docker-in-docker. start-services.sh runs `set -euo pipefail`, so the first Docker command (Postgres/Synapse) aborted the whole start when the daemon wasn't up yet — leaving a fresh Codespace with no services (host briefly up, Synapse down, no indexing). Wait for `docker info` before touching Docker. Also harden the launch (nohup) and log to a suspend-surviving path for diagnosis. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer/devcontainer.json | 2 +- .devcontainer/start-services.sh | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0c13082e6d..3b4f80d182 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -26,7 +26,7 @@ }, "postCreateCommand": "bash .devcontainer/setup.sh", - "postStartCommand": "setsid bash .devcontainer/start-services.sh > /tmp/start-services.log 2>&1 < /dev/null &", + "postStartCommand": "nohup setsid bash .devcontainer/start-services.sh > /workspaces/.boxel-start-services.log 2>&1 < /dev/null &", "customizations": { "codespaces": { diff --git a/.devcontainer/start-services.sh b/.devcontainer/start-services.sh index ab88f423a8..480b9eef4a 100755 --- a/.devcontainer/start-services.sh +++ b/.devcontainer/start-services.sh @@ -27,6 +27,16 @@ cd /workspaces/boxel # mise provides the pinned node/pnpm/ts-node toolchain. eval "$(mise activate bash)" +# postStartCommand fires this on every container start/resume, which races +# docker-in-docker coming up. Postgres and Synapse (below) run in Docker, and +# under `set -e` the first Docker command would abort the entire start if the +# daemon isn't ready yet — that's how a fresh Codespace ended up with the host +# briefly serving but Synapse never up and indexing dead. Wait for the daemon +# before touching it. +echo "==> Waiting for the Docker daemon..." +timeout 180 bash -c 'until docker info >/dev/null 2>&1; do sleep 2; done' \ + || echo "Warning: Docker daemon not ready after 180s; Postgres/Synapse may fail to start." + # Tear down any prior service instances before (re)starting. On a fresh # Codespace this is a no-op, but on a re-run (after a code pull, or a manual # restart) a still-running realm holds port 4201, so the new realm would hit