diff --git a/internal/fleet/start_phase.go b/internal/fleet/start_phase.go index 697c190c..6598f2ba 100644 --- a/internal/fleet/start_phase.go +++ b/internal/fleet/start_phase.go @@ -34,6 +34,10 @@ const ( // PhaseReconnecting: the attempt's connection dropped, and the next one is // due at RetryAt. PhaseReconnecting + // PhaseSeeding: the weights are still being fetched, so no instance has + // been launched yet and there is no boot to time — the count-up answers + // "how long has the fetch been going". + PhaseSeeding ) // StartPhase is a start's current situation. It carries no rendered text and no @@ -57,6 +61,7 @@ type StartPhase struct { const ( stateNoCapacity = "no-capacity" stateReady = "ready" + stateSeeding = "seeding" ) // RenderPhase is the phase's line at time now. A wait counts down towards @@ -72,6 +77,11 @@ func RenderPhase(p StartPhase, now time.Time) string { return "booting" } return "booting (" + formatPhaseDuration(now.Sub(p.Since)) + ")" + case PhaseSeeding: + if p.Since.IsZero() { + return "seeding the weights" + } + return "seeding the weights (" + formatPhaseDuration(now.Sub(p.Since)) + ")" case PhaseReconnecting: line := "connection dropped" if p.Detail != "" { @@ -163,8 +173,10 @@ func (t *startPhases) state(s string) { // connection: each described the attempt before it. It does not // supersede a boot — once a reply has reported the instance coming // up, the attempts that follow are polls of that same boot, and its - // elapsed time counts from the first reply that reported it. - if t.phase.Kind != PhaseBooting || !t.begun { + // elapsed time counts from the first reply that reported it. A + // seeding wait is the same shape: the attempts that follow a seeding + // reply poll the same fetch, and its clock keeps running. + if t.phase.Kind != PhaseBooting && t.phase.Kind != PhaseSeeding || !t.begun { t.enter(PhaseAttempting, "") } case s == stateReady: @@ -176,6 +188,11 @@ func (t *startPhases) state(s string) { // line remote.Start writes next, so the wait carries no due time // until that line arrives. t.enter(PhaseWaitingCapacity, s) + case s == stateSeeding: + // No instance exists yet, so this is not a boot: the fetch is what + // the count-up measures. enter() keeps Since on the first seeding + // reply, so repeated polls do not restart the clock. + t.enter(PhaseSeeding, s) default: t.enter(PhaseBooting, s) } diff --git a/internal/fleet/start_phase_test.go b/internal/fleet/start_phase_test.go index 0f345509..794de5a0 100644 --- a/internal/fleet/start_phase_test.go +++ b/internal/fleet/start_phase_test.go @@ -54,6 +54,54 @@ func TestRenderPhaseIsComputedAtDrawTime(t *testing.T) { } } +// A seeding reply is not a boot: no instance exists yet, and the line counts +// up from the first seeding reply. The attempts that follow it poll the same +// fetch, so they must not restart the clock. +func TestStartPhasesSeeding(t *testing.T) { + base := time.Date(2026, 9, 3, 12, 0, 0, 0, time.UTC) + + var got []StartPhase + progress, onState := StartPhases(func(p StartPhase) { got = append(got, p) }) + + onState(remote.StateInFlight) + onState(stateSeeding) + progress("seeding the weights (seed llamacpp--org-model--Q4_K_M); retrying in 60s") + // The next attempt supersedes nothing: it polls the fetch the seeding + // reply reported, and the reply that follows reports the same fetch. + onState(remote.StateInFlight) + onState(stateSeeding) + + want := []StartPhaseKind{PhaseAttempting, PhaseSeeding} + kinds := make([]StartPhaseKind, len(got)) + for i, p := range got { + kinds[i] = p.Kind + } + if len(kinds) != len(want) { + t.Fatalf("phases = %v, want %v", kinds, want) + } + for i := range want { + if kinds[i] != want[i] { + t.Fatalf("phases = %v, want %v", kinds, want) + } + } + if got[1].Detail != stateSeeding { + t.Errorf("seeding phase detail = %q, want %q", got[1].Detail, stateSeeding) + } + + // The rendered line counts up from the first seeding reply at two times. + seed := StartPhase{Kind: PhaseSeeding, Since: base, Detail: stateSeeding} + if line := RenderPhase(seed, base.Add(30*time.Second)); line != "seeding the weights (30s)" { + t.Errorf("RenderPhase(seed, +30s) = %q", line) + } + if line := RenderPhase(seed, base.Add(13*time.Minute)); line != "seeding the weights (13m 0s)" { + t.Errorf("RenderPhase(seed, +13m) = %q", line) + } + // A phase drawn without a clock renders its name alone, as a boot does. + if line := RenderPhase(StartPhase{Kind: PhaseSeeding, Detail: stateSeeding}, base); line != "seeding the weights" { + t.Errorf("RenderPhase(seed, no Since) = %q", line) + } +} + // The mapping from remote.Start's two callbacks onto phases: an attempt goes // out, is refused for capacity with a due time for the next one, and the // attempt that follows retires the refusal rather than leaving it standing diff --git a/internal/remote/remote.go b/internal/remote/remote.go index 217e1994..b7821108 100644 --- a/internal/remote/remote.go +++ b/internal/remote/remote.go @@ -326,6 +326,23 @@ var startRetryWait = 5 * time.Second // underway, not a capacity wait. const StateInFlight = "in-flight" +// stateSeeding is the control plane's word for "the weights are still being +// fetched" — no instance has been launched yet, so there is no boot to wait +// on, and the reply's seedId is what an operator follows to see how far the +// fetch has got. +const stateSeeding = "seeding" + +// giveUpWaiting is Start's error when the caller's deadline expires mid-wait. +// When the last reply was the seeding state, the useful next steps are +// following the seed and resuming the wait, so the error carries both. +func giveUpWaiting(ctx context.Context, state, seedID string) error { + if state == stateSeeding && seedID != "" { + return fmt.Errorf("gave up waiting for the endpoint: the weights are still seeding (seed %s) — follow it with `spinloop remote seed status %s`, and re-run start with a longer --timeout: %w", + seedID, seedID, ctx.Err()) + } + return fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err()) +} + // Start boots the instance and blocks until the model is serving, retrying // while the endpoint reports it is still starting. progress is called with a // status line before each wait. onState, when non-nil, is called with the raw @@ -354,6 +371,11 @@ func Start(ctx context.Context, cfg Config, progress func(string), onState func( startURL = u.String() } } + // The last reply's state and seed id, for the give-up error: a deadline + // that expires mid-seed is not the same situation as one that expires + // mid-boot, and the operator should be told which. + lastState := "" + lastSeedID := "" for { // Supersedes whatever the previous attempt reported — including a // no-capacity reply: this attempt has not refused anything yet, and a @@ -369,7 +391,7 @@ func Start(ctx context.Context, cfg Config, progress func(string), onState func( progress(fmt.Sprintf("connection dropped (%v); retrying in %s", urlErr.Unwrap(), startRetryWait)) select { case <-ctx.Done(): - return nil, fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err()) + return nil, giveUpWaiting(ctx, lastState, lastSeedID) case <-time.After(startRetryWait): } continue @@ -383,14 +405,21 @@ func Start(ctx context.Context, cfg Config, progress func(string), onState func( case resp.StatusCode == http.StatusOK && resp.State == "ready": return resp, nil case resp.StatusCode == http.StatusServiceUnavailable: + lastState, lastSeedID = resp.State, resp.SeedID wait := resp.RetryAfterSeconds if wait <= 0 { wait = 1 } - progress(fmt.Sprintf("instance %s; retrying in %ds", resp.State, wait)) + // "instance " would be a lie while the weights seed: there + // is no instance yet, and the seed is what the operator follows. + if resp.State == stateSeeding && resp.SeedID != "" { + progress(fmt.Sprintf("seeding the weights (seed %s); retrying in %ds", resp.SeedID, wait)) + } else { + progress(fmt.Sprintf("instance %s; retrying in %ds", resp.State, wait)) + } select { case <-ctx.Done(): - return nil, fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err()) + return nil, giveUpWaiting(ctx, lastState, lastSeedID) case <-time.After(time.Duration(wait) * time.Second): } default: diff --git a/internal/remote/remote_test.go b/internal/remote/remote_test.go index 964675ed..05be718f 100644 --- a/internal/remote/remote_test.go +++ b/internal/remote/remote_test.go @@ -192,6 +192,131 @@ func TestStart_RetriesUntilReady(t *testing.T) { } } +// A 503 seeding reply names the seed on every progress line — there is no +// instance to speak of yet — and the loop keeps polling until the weights +// are in and the model is serving. +func TestStart_NamesTheSeedWhileSeeding(t *testing.T) { + stubAWSEnv(t) + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + if calls < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"state":"seeding","seedId":"llamacpp--org-model--Q4_K_M","retry_after_seconds":0}`)) + return + } + w.Write([]byte(`{"state":"ready","base_url":"http://198.51.100.1:8000/v1","api_key":"sk-test"}`)) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + var progress []string + resp, err := Start(context.Background(), cfg, func(msg string) { progress = append(progress, msg) }, nil, nil) + if err != nil { + t.Fatal(err) + } + if resp.State != "ready" { + t.Errorf("unexpected response: %+v", resp) + } + if calls != 3 { + t.Errorf("expected 3 calls, got %d", calls) + } + for _, line := range progress { + if !strings.Contains(line, "seeding the weights") || !strings.Contains(line, "llamacpp--org-model--Q4_K_M") { + t.Errorf("progress line does not name the seed: %q", line) + } + } +} + +// A deadline that expires mid-seed is not the generic give-up: the weights +// are still being fetched, and the error carries the seed's follow command +// and the longer-timeout hint. +func TestStart_GiveUpDuringSeedingNamesTheSeed(t *testing.T) { + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"state":"seeding","seedId":"llamacpp--org-model--Q4_K_M","retry_after_seconds":30}`)) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _, err := Start(ctx, cfg, func(string) {}, nil, nil) + if err == nil { + t.Fatal("expected an error when the deadline expires") + } + for _, want := range []string{"seeding", "llamacpp--org-model--Q4_K_M", "spinloop remote seed status", "--timeout"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("give-up error does not name %q: %v", want, err) + } + } +} + +// The last reply was seeding and then the connection dropped: the give-up +// that follows the drop still names the seed, so the state a reply carried +// survives a dropped connection. +func TestStart_GiveUpAfterADroppedConnectionNamesTheSeed(t *testing.T) { + stubAWSEnv(t) + origWait := startRetryWait + startRetryWait = time.Second + t.Cleanup(func() { startRetryWait = origWait }) + + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + if calls == 1 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"state":"seeding","seedId":"llamacpp--org-model--Q4_K_M","retry_after_seconds":0}`)) + return + } + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + conn.Close() + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + ctx, cancel := context.WithTimeout(context.Background(), 1800*time.Millisecond) + defer cancel() + _, err := Start(ctx, cfg, func(string) {}, nil, nil) + if err == nil { + t.Fatal("expected an error once the deadline passed") + } + for _, want := range []string{"seeding", "llamacpp--org-model--Q4_K_M", "spinloop remote seed status"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("give-up error does not name %q: %v", want, err) + } + } +} + +// A deadline that expires outside seeding keeps today's give-up message. +func TestStart_GiveUpOutsideSeedingIsGeneric(t *testing.T) { + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"state":"starting","retry_after_seconds":30}`)) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _, err := Start(ctx, cfg, func(string) {}, nil, nil) + if err == nil { + t.Fatal("expected an error when the deadline expires") + } + if want := "gave up waiting for the endpoint: context deadline exceeded"; err.Error() != want { + t.Errorf("give-up error = %q, want %q", err.Error(), want) + } +} + // onState must see both the raw state of every poll and each attempt as it is // issued, so a caller can tell a capacity wait apart from a boot rather than // assume the instance is starting. diff --git a/openspec/changes/archive/2026-09-06-guard-start-during-seed/.openspec.yaml b/openspec/changes/archive/2026-09-06-guard-start-during-seed/.openspec.yaml new file mode 100644 index 00000000..1a62d62b --- /dev/null +++ b/openspec/changes/archive/2026-09-06-guard-start-during-seed/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-06 diff --git a/openspec/changes/archive/2026-09-06-guard-start-during-seed/design.md b/openspec/changes/archive/2026-09-06-guard-start-during-seed/design.md new file mode 100644 index 00000000..7cf140f8 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-guard-start-during-seed/design.md @@ -0,0 +1,201 @@ +## Context + +See proposal.md for the motivation. The state this design works with: + +- A seed's completeness is the `_seed.json` manifest the seeder writes as its + final step; `weightsPresent(bucket, cfg)` in `remote/lambda/shared/seed.ts` + answers "are these weights done" from it, including the companion check. + This is the only judge of completeness the control plane has. +- A seed instance is discoverable by tag: `cloud-vm-llm: seed` plus + `cloud-vm-llm:seed-id: `, where `` is derived from + (runner, modelId, quant) by `seedIdFor` — the same inputs the start Lambda + already holds in the environment's deploy-config. +- `launchSeedInstance` in `remote/lambda/shared/seed/launch.ts` is the one + launch path: a deterministic idempotency token converges concurrent requests + onto one instance, and a dead idempotency hit is escaped with a fresh + generation. The deploy Lambda already calls it exactly this way, with the + seed infrastructure read from its own environment (`seedInfraFromEnv`). +- `findManagedInstances` returns pending, running *and stopped* instances + (stopped has to stay in for the endpoint re-wake path), so "a seed instance + exists" is not the same as "a seed is in flight". +- The Go start client (`internal/remote.Start`) already loops on any 503 it + gets, sleeping the reply's `retry_after_seconds`, until the context + deadline. Both `spinloop remote start` and the fleet wake draw their + progress line from `fleet.StartPhases` / `fleet.RenderPhase`. + +## Goals / Non-Goals + +**Goals:** + +- A start never launches or re-wakes an instance whose weights are absent. +- While the weights are absent, a start either re-attaches to the running + seed or starts one itself, and reports a retryable state naming the seed. +- One wording for the seeding state across `remote start` and the fleet + dashboard, built the same way as every other state. +- No new commands, flags, or control-plane endpoints. + +**Non-Goals:** + +- `spinloop remote status` is unchanged: seed visibility stays on + `seed ls` / `seed status`. +- No re-seeding of weights that *are* present — that remains the deliberate + `--reseed` / `seed start --force` path. +- The streaming-Lambda vs EC2 thread from #97 is a separate decision. +- Deploy's auto-seed currently skips the concurrency cap; making it enforce + the cap is a follow-up, not part of this change. + +## Decisions + +### 1. The gate checks the manifest, inside the wake path, before any instance work + +`wake()` in `remote/lambda/start/index.ts` gains one check after it has read +the deploy-config and found the environment's EIP and security group, and +before it looks for an existing instance: if `weightsPresent(WEIGHTS_BUCKET, +deployConfig)` is false, the wake does not proceed to launch or re-wake. + +The manifest is the right judge because it is the only record that says the +weights are *complete*: a seed's CloudWatch records stop arriving when its +process dies, and the absence of an error says nothing. Checking the seed's +state alone would answer "is a seed working on this" but not "is the work +done"; checking the prefix's contents would re-derive what the manifest +records. Placement after the deploy-config read is forced (the check needs +runner, model, quant and the companion map), and after the EIP/security-group +check keeps the existing "run `spinloop remote deploy`" answers for +never-deployed environments in front of anything about weights. + +### 2. The start launches the seed itself, through the shared launch path + +When the weights are absent and no seed is in flight, the start calls +`launchSeedInstance(buildSeedJob(deployConfig, infra, ''), infra)` — the same +two calls deploy makes, with the environment's full deploy-config so +companions ride along. The alternatives: + +- **Refuse with "run `spinloop remote deploy`"** keeps start mutation-free, + but makes start fail in a state only deploy can fix, for an environment + that is already deployed — and fleet wakes have no operator standing by to + run anything. It also leaves the adjacent case broken: a failed seed leaves + a partial prefix in the bucket, and a plain launch against it is exactly + the footgun this change closes. +- **Invoke the seed Lambda** would reuse its validation and cap, at the cost + of a second wire format (parsing the seed Lambda's JSON reply inside the + start Lambda) and an IAM edge between two Lambdas, for logic that is + already a shared, tested module the deploy Lambda imports directly. + +Auto-launching also answers #97's open question "auto-retry a stalled/failed +seed on next deploy/start, or require explicit `--force-reseed`": the answer +is auto-retry on the next start (deploy already has these semantics, so the +two paths agree), with the failure still diagnosable via +`spinloop remote seed status `. A failed seed's dead instance is handled +by the launch path's existing escape: the constant `auto` idempotency token +returns the terminated instance, the launch sees a dead state, and retries +with a fresh generation — one new seed, converged. + +A launch *failure* (for example, no capacity for the seed instance type) +returns the same retryable seeding state with the error in the message, +rather than a fatal 502 as deploy gives: a transient EC2 refusal of the seed +should not kill the start that wanted the model served, and the next poll +retries the launch. The convergence token means a retry can never double the +compute. + +### 3. "In flight" means a tagged instance that is pending or running + +The start filters seed instances by seed id and counts only `pending` and +`running` states as in flight — the same liveness rule the seed status join +uses. `findManagedInstances` includes `stopped`, and a stopped seed instance +is a dead seed (its joined state is already failed); treating it as in flight +would wedge every start for those weights behind a body that runs nothing. +The discovery filter moves from `remote/lambda/seed/index.ts` into +`remote/lambda/shared/seed/` so the seed and start Lambdas share one +definition of "find the seed instances". + +Consulting CloudWatch records is deliberately not part of this check: +instance existence and state decide whether to block and whether to launch; +the join of records with existence is what `seed status` reports, and +re-deriving it per wake would add a log-group read to a hot path for a +verdict the instance state already gives. + +### 4. The reply is a 503 `seeding` state, in the existing shape + +``` +503 { state: 'seeding', seedId: '', retry_after_seconds: 60, + message: 'seeding the weights — follow it with `spinloop remote seed status `' } +``` + +The Go client loops on any 503 it receives, so no client control-flow change +is needed — only wording. `retry_after_seconds` of 60 matches the start +Lambda's existing waits; a seed runs for minutes, and polling faster buys +nothing. The `seedId` rides the reply (the `Response` struct already carries +it for deploy) so the CLI can name it without a second call. + +The cap is enforced before launching: a start that finds at least +`MAX_CONCURRENT_SEEDS` seeds in flight returns the seeding state with a +message saying the cap was reached, and the next poll retries. That is a +queue, not a bypass — the cap still bounds compute, and the reply says what +is going on. + +### 5. One new phase, drawn by both surfaces + +`internal/fleet/start_phase.go` gains a `PhaseSeeding` kind, entered when +`onState` reports `seeding`, and rendered as `seeding the weights (Xm Ys)` +counting up from the first seeding reply — the same shape as `booting`, +since the existing progress-line parser only attaches a due time to a +capacity wait and a counting-up phase already answers "how long has this +been going on". `internal/remote.Start`'s 503 progress line special-cases +the state: the generic line reads `instance `, and there is no +instance while the weights seed, so the seeding line names the seed instead. +If the context deadline expires, the give-up error names the state it last +saw; when that is seeding, it carries the seed's follow command and the +hint that a longer `--timeout` resumes the wait. The default 15-minute +timeout is unchanged: a seed takes 15–20 minutes, so a start that begins +one can outlast the default, and the honest answer is a clear message and a +safe re-run rather than silently waiting longer for everyone. + +### 6. CDK: the start function gets the seed environment and three permissions + +`remote/lib/llm-stack.ts` already assembles a `seedEnv` object — "everything +a Lambda needs to launch and supervise seeds" — currently spread into the +deploy and seed functions. The start function's environment gains +`...seedEnv` plus `MAX_CONCURRENT_SEEDS`, and its role gains the three +statements it lacks for the seed launch: `ssm:GetParameter` on the stock +AL2023 AMI parameter, `iam:PassRole` on the seed instance profile's role, +and read on the weights bucket's `models/*` prefix (for the manifest check; +the start Lambda has no weights-bucket grant today — the boot's S3 sync runs +under the instance profile, not the Lambda). `ec2:RunInstances`, +`ec2:CreateTags` and `ec2:DescribeInstances` are already granted. Code and +IAM ship together in one `remote bootstrap`, so there is no state where the +gate exists without its permissions. + +## Risks / Trade-offs + +- **A seed outlives the default start timeout** (15–20 min seed + several + minutes boot > 15 min default) → the give-up message names the seed and + the follow command; re-running start is safe (it re-attaches, converging + on the same seed, and never launches a second instance); the operator can + also just run `spinloop remote seed status ` and start later. +- **A persistently failing seed (bad model id, no HF access) is re-run on + every start** → each attempt is bounded by the maximum seed lifetime and + the failure is recorded where `seed status` reads it; the start's reply + and the CLI line both name the seed while it runs. Deploy already has + identical re-run semantics, so this changes no behaviour the account has + not already accepted. +- **The start Lambda gains EC2 mutation scope it did not have** (it can now + run seed instances) → the grants are the same ones the deploy and seed + Lambdas hold: the instance profile is pass-able only to EC2, and the + launch only ever happens for the seed id derived from the environment's + own deploy-config, with the same convergence token deploy uses. +- **Eventual consistency between the manifest write and the next wake** → a + wake that reads the manifest a heartbeat before the seeder's final write + sees absent, re-attaches to the still-alive seed, and the next poll reads + present. The seed writes the manifest only after every file is complete, + so a wake can never observe "present but partial". +- **A stopped seed instance lingers past the sweep** → not counted as in + flight, so a start launches a fresh seed rather than waiting on it; the + sweep's existing seed pass still reaps the leftover. + +## Migration Plan + +Ship the CDK change with the Lambda code, then `spinloop remote bootstrap` +redeploys the control plane (code and IAM in one apply). No data migration: +the gate reads state that already exists (the manifest, the seed tags). +Rollback is a bootstrap with the previous version — the gate simply stops +being there and behaviour returns to today's. diff --git a/openspec/changes/archive/2026-09-06-guard-start-during-seed/proposal.md b/openspec/changes/archive/2026-09-06-guard-start-during-seed/proposal.md new file mode 100644 index 00000000..e306aaa2 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-guard-start-during-seed/proposal.md @@ -0,0 +1,64 @@ +## Why + +`spinloop remote start` never checks whether the environment's weights are in +S3. Run while the model's seed is still transferring, it launches the runtime +instance, which syncs the *partial* prefix the seed has written so far and +boots against incomplete weights — the footgun the deploy reply's warning +describes, with no guard behind it (gap 3 of #97, the one the seed rework left +open). The same is true after a seed has failed: the partial prefix is still in +the bucket, and a start happily launches against it. + +## What Changes + +- The start path checks that the environment's weights are present (the + manifest the seeder writes as its final step) before it launches or re-wakes + the environment's instance. +- While the weights are absent, a start launches nothing. It reports a + retryable `seeding` state naming the seed — the same 503/retry-after shape as + the existing `starting` and `no-capacity` states — so the existing polling + loop waits instead of the instance booting on an incomplete prefix. +- If no seed is already running for those weights, the start launches one + itself, through the same shared launch path deploy uses: the deterministic + idempotency token still converges concurrent requests onto one instance, and + the same cap on seeds in flight applies. A failed or never-run seed is + therefore re-run on the next start, and a start interrupted mid-seed simply + re-attaches and keeps waiting. +- The CLI recognises the `seeding` state: `spinloop remote start` (and the + fleet wake, which reuses the same start surface) shows a line that names the + seed, and a timeout that expires mid-seed says how to follow the seed and + how to resume waiting rather than a generic "gave up". +- The start Lambda gains the environment, permissions and read access to the + weights bucket it needs to make the check and launch a seed. + +No new commands or flags: `spinloop remote seed` is unchanged, and `start` +keeps its existing interface. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `endpoint-lifecycle`: "Starting on demand" gains a precondition — a start + does not launch an instance whose weights are not yet in S3; it reports a + retryable seeding state that names the seed, and it ensures a seed is in + flight when one is not. + +## Impact + +- `remote/lambda/start/index.ts` — the weights gate in the wake path. +- `remote/lambda/seed/index.ts` + `remote/lambda/shared/seed/` — the seed + instance discovery the seed Lambda has inlined moves to the shared seed + modules so the start path uses the same one. +- `remote/lib/llm-stack.ts` — the start function's environment, IAM (launching + and supervising seeds, reading the weights bucket) and, for tests, the + matching CDK assertions. +- `internal/remote/remote.go` — the start client's handling of the `seeding` + state and its timeout message. +- `internal/fleet/start_phase.go` — a phase for the seeding state, so the + dashboard and `remote start` word it the same way as every other state. +- Existing deployments need a `spinloop remote bootstrap` after the new CDK + ships for the start Lambda to gain its permissions; until then the gate is + inert and behaviour is unchanged. diff --git a/openspec/changes/archive/2026-09-06-guard-start-during-seed/specs/endpoint-lifecycle/spec.md b/openspec/changes/archive/2026-09-06-guard-start-during-seed/specs/endpoint-lifecycle/spec.md new file mode 100644 index 00000000..94743cd5 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-guard-start-during-seed/specs/endpoint-lifecycle/spec.md @@ -0,0 +1,124 @@ +## MODIFIED Requirements + +### Requirement: Starting on demand + +Each environment SHALL hold no running instance when idle. A start request names +an environment and SHALL launch that environment's instance, trying each +configured availability zone in turn until one has capacity, since GPU capacity +is not guaranteed in any single zone. A launch SHALL provision the instance's +root volume: a gp3 volume of the AMI's own root size, with provisioned +throughput at the volume's ceiling — the size is read from the AMI's own root +mapping, because a launch's block device mapping replaces the AMI's rather +than extending it — and IOPS provisioned at four times that throughput, +which is the minimum EC2 allows for it (gp3 caps throughput at 0.25 MiB/s +per provisioned IOP). The +instance SHALL be given the environment's own stable address (its Elastic IP) +so the environment's URL does not change between launches, and the request +SHALL NOT report success until the model is answering — the caller receives +one "ready", never a URL that is not yet serving. When no capacity can be +found anywhere, the response SHALL say so and SHALL be retryable rather than +fatal. One shared set of lifecycle Lambdas SHALL serve every environment in +the account, selecting the instance by the environment identifier. + +Before launching or re-waking the instance, a start SHALL check that the +environment's weights are present in shared storage, judged by the same +completeness record the seeding writes rather than by the absence of an error. +While the weights are absent, a start SHALL NOT launch or re-wake the +instance, and SHALL report a retryable state that names the seed producing +the weights, so a caller can wait for the seed or follow it separately rather +than receiving an instance that boots against an incomplete prefix. When no +seed is running for those weights, the start SHALL start one, so that the +weights are produced rather than the start failing; a seed whose compute has +ceased — failed, stopped or reaped — does not count as running, and its +re-run follows the same identity and convergence rules as any other seed +request. A start that would exceed the cap on seeds in flight SHALL NOT start +another seed, and SHALL report the retryable state until a later start can. + +The control plane SHALL request the engine's start on every path — a fresh +launch and a re-wake alike — once the instance's daemon answers its control +API, which on a fresh boot is the signal that the boot has stored the deploy +config; the boot's own user data SHALL NOT start the engine. The start SHALL +carry the deploy config as its body, so it always names the exact config the +daemon runs. + +#### Scenario: A zone without capacity is not the end of it + +- **WHEN** the first availability zone cannot provide the instance type +- **THEN** the remaining zones are tried before reporting failure + +#### Scenario: Ready means serving + +- **WHEN** a start request returns success +- **THEN** the model is answering requests at the environment's reported address + +#### Scenario: No capacity anywhere + +- **WHEN** every configured zone is out of capacity +- **THEN** the response says so and indicates the caller may retry shortly + +#### Scenario: Starting the right environment + +- **WHEN** several environments are deployed and a start names one of them +- **THEN** only that environment's instance is launched, at its own Elastic IP + +#### Scenario: Nothing has been deployed + +- **WHEN** a start is requested for an environment before it has been deployed +- **THEN** it fails saying what to deploy, rather than launching an instance + with nothing to serve + +#### Scenario: A launch provisions the root volume + +- **WHEN** a start launches a fresh instance +- **THEN** its root volume is the AMI's gp3 root, at the AMI's own size, with + provisioned throughput at the volume's ceiling and provisioned IOPS at four + times that throughput + +#### Scenario: The control plane starts the engine on a fresh boot + +- **WHEN** a fresh instance's daemon first answers its control API +- **THEN** the start request itself issues the engine's start, with the + deploy config as its body, and reports ready only once the model answers — + the boot started no engine + +#### Scenario: A start while the weights are seeding launches nothing + +- **WHEN** a start is requested for an environment whose weights are still + being seeded +- **THEN** no instance of the environment is launched or re-woken, and the + response is retryable and names the running seed + +#### Scenario: A start with absent weights and no running seed starts one + +- **WHEN** a start is requested for an environment whose weights are absent + and no seed is running for them +- **THEN** a seed for those weights is started, no instance is launched, and + the response is the same retryable state naming the seed + +#### Scenario: Two starts for the same weights share one seed + +- **WHEN** starts for two environments naming the same weights arrive while no + seed is running +- **THEN** one seed is started, and both responses name it + +#### Scenario: A ceased seed does not block a start + +- **WHEN** the only seed for the weights has ceased — failed, stopped or + reaped — and a start is requested +- **THEN** the weights are treated as absent: a new seed is started and the + response is the retryable state, never a launch against the partial prefix + the failed seed left behind + +#### Scenario: A start that would exceed the seed cap waits + +- **WHEN** the cap on seeds in flight is reached and a start needs to start a + seed for absent weights +- **THEN** no further seed is started, and the response is retryable until a + later start can start the seed + +#### Scenario: A start proceeds once the weights are present + +- **WHEN** a start has been reported in the seeding state and the seed has + since finished, leaving the weights present +- **THEN** the next start launches the environment's instance and proceeds to + ready as usual diff --git a/openspec/changes/archive/2026-09-06-guard-start-during-seed/tasks.md b/openspec/changes/archive/2026-09-06-guard-start-during-seed/tasks.md new file mode 100644 index 00000000..356452e0 --- /dev/null +++ b/openspec/changes/archive/2026-09-06-guard-start-during-seed/tasks.md @@ -0,0 +1,29 @@ +## 1. Shared seed discovery + +- [x] 1.1 Add `findSeedInstances(seedId?)` to `remote/lambda/shared/seed/` (a new `discovery.ts`), filtering managed instances by the seed tag and, when given, the seed-id tag, and export it. Verify `remote/lambda/seed/index.ts` imports it and drops its local `findSeeds`, and `pnpm test` in `remote/` passes. + +## 2. The start Lambda's weights gate + +- [x] 2.1 In `remote/lambda/start/index.ts`'s `wake()`, after the deploy-config and EIP/security-group checks and before the existing-instance lookup, check `weightsPresent(WEIGHTS_BUCKET, deployConfig)`; when true, proceed unchanged. Verify the existing start Lambda tests still pass with the check stubbed present. +- [x] 2.2 When the weights are absent, return the 503 `seeding` reply (`state`, `seedId` from `seedIdFor`, `retry_after_seconds`, a message naming `spinloop remote seed status `) when a seed instance for that id is pending or running — stopped instances do not count. Verify a new vitest case asserts no `RunInstances` call is made and the reply carries the seed id. +- [x] 2.3 When no seed is in flight, enforce the in-flight cap (return the `seeding` reply with a cap message, no launch) and otherwise call `launchSeedInstance(buildSeedJob(deployConfig, infra, ''), infra)` with the environment's full deploy-config, returning the `seeding` reply. Verify new vitest cases: cap reached launches nothing; under the cap the launch receives the deploy-config's companions; the reply carries the launched seed id. +- [x] 2.4 Treat a seed launch failure (e.g. no capacity for the seed instance type) as the retryable `seeding` reply with the error in the message, not a fatal 502. Verify a new vitest case stubs the launch to throw and asserts a 503 `seeding` reply. + +## 3. The Go start client + +- [x] 3.1 In `internal/remote.Start`, special-case a 503 whose state is `seeding`: the progress line names the seed (`seedId` from the reply) instead of reading `instance `. Verify a new test in `internal/remote` drives a seeding 503 and asserts the line names the seed and the loop continues until ready. +- [x] 3.2 When the context deadline expires, make the give-up error name the state last seen; when that state is seeding, carry the seed's follow command and the longer-`--timeout` hint. Verify a new test asserts the error text for a deadline reached during seeding, and that non-seeding deadlines keep today's message. + +## 4. The fleet start phase + +- [x] 4.1 In `internal/fleet/start_phase.go`, add a `PhaseSeeding` kind entered when `onState` reports `seeding`, rendered by `RenderPhase` as `seeding the weights (Xm Ys)` counting up from the first seeding reply. Verify `internal/fleet` tests cover the state mapping and the rendered line at two different `now` values. + +## 5. CDK + +- [x] 5.1 In `remote/lib/llm-stack.ts`, give the start function's environment `...seedEnv` and `MAX_CONCURRENT_SEEDS`, and its role the three grants it lacks for the seed launch: `ssm:GetParameter` on the stock AL2023 AMI parameter, `iam:PassRole` on the seed instance profile's role, and read on the weights bucket's `models/*` prefix. Verify `pnpm build` and `pnpm test` pass in `remote/` and `pnpm synth` produces a template whose StartFn policy includes the new statements. + +## 6. Verification + +- [x] 6.1 Run the Go suite with coverage (`go test ./... -cover`), `go vet ./...` and `gofmt -l .`, and confirm total coverage stays at or above 80% with no vet or formatting findings. +- [x] 6.2 Run the remote TypeScript suite (`pnpm test` in `remote/`) and confirm every new seed-gate, start-client and phase test passes. +- [x] 6.3 Walk the spec scenarios in `specs/endpoint-lifecycle/spec.md` against the new tests and confirm each one is exercised by at least one test case or explicitly noted in the PR description where it needs a live account. diff --git a/openspec/specs/endpoint-lifecycle/spec.md b/openspec/specs/endpoint-lifecycle/spec.md index 96e5bb32..31e4bf4f 100644 --- a/openspec/specs/endpoint-lifecycle/spec.md +++ b/openspec/specs/endpoint-lifecycle/spec.md @@ -26,6 +26,20 @@ found anywhere, the response SHALL say so and SHALL be retryable rather than fatal. One shared set of lifecycle Lambdas SHALL serve every environment in the account, selecting the instance by the environment identifier. +Before launching or re-waking the instance, a start SHALL check that the +environment's weights are present in shared storage, judged by the same +completeness record the seeding writes rather than by the absence of an error. +While the weights are absent, a start SHALL NOT launch or re-wake the +instance, and SHALL report a retryable state that names the seed producing +the weights, so a caller can wait for the seed or follow it separately rather +than receiving an instance that boots against an incomplete prefix. When no +seed is running for those weights, the start SHALL start one, so that the +weights are produced rather than the start failing; a seed whose compute has +ceased — failed, stopped or reaped — does not count as running, and its +re-run follows the same identity and convergence rules as any other seed +request. A start that would exceed the cap on seeds in flight SHALL NOT start +another seed, and SHALL report the retryable state until a later start can. + The control plane SHALL request the engine's start on every path — a fresh launch and a re-wake alike — once the instance's daemon answers its control API, which on a fresh boot is the signal that the boot has stored the deploy @@ -73,6 +87,48 @@ daemon runs. deploy config as its body, and reports ready only once the model answers — the boot started no engine +#### Scenario: A start while the weights are seeding launches nothing + +- **WHEN** a start is requested for an environment whose weights are still + being seeded +- **THEN** no instance of the environment is launched or re-woken, and the + response is retryable and names the running seed + +#### Scenario: A start with absent weights and no running seed starts one + +- **WHEN** a start is requested for an environment whose weights are absent + and no seed is running for them +- **THEN** a seed for those weights is started, no instance is launched, and + the response is the same retryable state naming the seed + +#### Scenario: Two starts for the same weights share one seed + +- **WHEN** starts for two environments naming the same weights arrive while no + seed is running +- **THEN** one seed is started, and both responses name it + +#### Scenario: A ceased seed does not block a start + +- **WHEN** the only seed for the weights has ceased — failed, stopped or + reaped — and a start is requested +- **THEN** the weights are treated as absent: a new seed is started and the + response is the retryable state, never a launch against the partial prefix + the failed seed left behind + +#### Scenario: A start that would exceed the seed cap waits + +- **WHEN** the cap on seeds in flight is reached and a start needs to start a + seed for absent weights +- **THEN** no further seed is started, and the response is retryable until a + later start can start the seed + +#### Scenario: A start proceeds once the weights are present + +- **WHEN** a start has been reported in the seeding state and the seed has + since finished, leaving the weights present +- **THEN** the next start launches the environment's instance and proceeds to + ready as usual + ### Requirement: Stopping when unused A running instance SHALL be **stopped**, not terminated, once unused, so that the boot disk and synced weights are preserved for fast re-wake. After a further configured period in the stopped state without a start request, the instance SHALL be **terminated** to free storage. Activity SHALL be judged from the inference server's own counters, read on the instance, and SHALL account for both requests in flight and work that started and finished between two readings. Because the metric names differ per inference engine, the check SHALL read the names belonging to the engine that is deployed. diff --git a/remote/lambda/seed/index.ts b/remote/lambda/seed/index.ts index 1d4490ab..3c63c78e 100644 --- a/remote/lambda/seed/index.ts +++ b/remote/lambda/seed/index.ts @@ -9,13 +9,7 @@ */ import type { LambdaFunctionURLEvent, LambdaFunctionURLResult } from 'aws-lambda'; -import { - errorName, - findManagedInstances, - requireEnv, - terminateInstance, - type InstanceInfo, -} from '../shared/aws'; +import { errorName, requireEnv, terminateInstance } from '../shared/aws'; import { isRunner, LATEST_SPINLOOP, @@ -25,25 +19,13 @@ import { import { jsonResponse } from '../shared/http'; import { weightsPresent } from '../shared/seed'; import { buildSeedJob, launchSeedInstance, seedInfraFromEnv } from '../shared/seed/launch'; -import { - SEED_ID_TAG_KEY, - SEED_TAG_VALUE, - seedIdFor, -} from '../shared/seed/identity'; +import { SEED_ID_TAG_KEY, SEED_MODEL_TAG_KEY, seedIdFor } from '../shared/seed/identity'; +import { findSeedInstances } from '../shared/seed/discovery'; import { readSeedStatus, writeTerminalRecord } from '../shared/seed/status'; const TAG_KEY = requireEnv('TAG_KEY'); const MAX_CONCURRENT_SEEDS = Number(requireEnv('MAX_CONCURRENT_SEEDS')); -/** Live seed instances, optionally narrowed to one seed id. */ -async function findSeeds(seedId?: string): Promise { - return findManagedInstances( - TAG_KEY, - SEED_TAG_VALUE, - seedId ? [{ Name: `tag:${SEED_ID_TAG_KEY}`, Values: [seedId] }] : [], - ); -} - export async function handler(event: LambdaFunctionURLEvent): Promise { const method = event.requestContext?.http?.method ?? 'POST'; const seedId = event.queryStringParameters?.id; @@ -102,7 +84,7 @@ async function start(event: LambdaFunctionURLEvent): Promise 0) { return jsonResponse(200, { seedId, @@ -142,7 +124,7 @@ async function start(event: LambdaFunctionURLEvent): Promise= MAX_CONCURRENT_SEEDS) { return jsonResponse(429, { error: `${inFlight.length} seeds are already running (cap ${MAX_CONCURRENT_SEEDS}) — wait for one to finish`, @@ -170,7 +152,7 @@ async function start(event: LambdaFunctionURLEvent): Promise { - const instances = await findSeeds(seedId); + const instances = await findSeedInstances(TAG_KEY, seedId); const result = await readSeedStatus(seedId, instances[0] ?? null); // A seed nobody has ever run has neither an instance nor records; say so // rather than reporting it as a failure. @@ -182,7 +164,7 @@ async function status(seedId: string): Promise { /** GET — every seed in flight. */ async function list(): Promise { - const instances = await findSeeds(); + const instances = await findSeedInstances(TAG_KEY); const seeds = await Promise.all( instances.map(async (instance) => { const id = instance.tags?.[SEED_ID_TAG_KEY] ?? ''; @@ -190,7 +172,7 @@ async function list(): Promise { return { seedId: id, instanceId: instance.instanceId, - modelId: instance.tags?.['cloud-vm-llm:seed-model'], + modelId: instance.tags?.[SEED_MODEL_TAG_KEY], state: detail?.state ?? 'starting', progressPercent: detail?.progressPercent, startedAt: instance.launchTime?.toISOString(), @@ -210,7 +192,7 @@ async function list(): Promise { * safe, and the caller's intent is satisfied either way. */ async function stop(seedId: string): Promise { - const instances = await findSeeds(seedId); + const instances = await findSeedInstances(TAG_KEY, seedId); if (instances.length === 0) { return jsonResponse(200, { seedId, stopped: false, message: `no seed ${JSON.stringify(seedId)} is running` }); } diff --git a/remote/lambda/shared/seed/discovery.ts b/remote/lambda/shared/seed/discovery.ts new file mode 100644 index 00000000..64d7b5af --- /dev/null +++ b/remote/lambda/shared/seed/discovery.ts @@ -0,0 +1,24 @@ +/** + * Finding seed instances by identity. + * + * The seed Lambda uses this for its join, list and stop paths, and the start + * Lambda's weights gate uses it to find the seed for the weights an + * environment would sync — one filter, so the two Lambdas agree on which + * instance is a seed's compute. + * + * The results still carry stopped instances: the caller decides what "in + * flight" means, and the seed's joined-state logic and the start's gate count + * only pending and running, the same way. + */ + +import { findManagedInstances, type InstanceInfo } from '../aws'; +import { SEED_ID_TAG_KEY, SEED_TAG_VALUE } from './identity'; + +/** Seed instances, optionally narrowed to one seed id. */ +export function findSeedInstances(tagKey: string, seedId?: string): Promise { + return findManagedInstances( + tagKey, + SEED_TAG_VALUE, + seedId ? [{ Name: `tag:${SEED_ID_TAG_KEY}`, Values: [seedId] }] : [], + ); +} diff --git a/remote/lambda/start/index.ts b/remote/lambda/start/index.ts index 21cb4ec6..d170939e 100644 --- a/remote/lambda/start/index.ts +++ b/remote/lambda/start/index.ts @@ -36,6 +36,10 @@ import { } from '../shared/environments'; import { DAEMON_STATUS_CMD, parseDaemonStatus } from '../shared/daemon'; import { jsonResponse } from '../shared/http'; +import { weightsPresent } from '../shared/seed'; +import { findSeedInstances } from '../shared/seed/discovery'; +import { seedIdFor } from '../shared/seed/identity'; +import { buildSeedJob, launchSeedInstance, seedInfraFromEnv } from '../shared/seed/launch'; import { DAEMON_CONFIG_DIR, runnerSpec } from '../runners'; const TAG_KEY = requireEnv('TAG_KEY'); @@ -48,6 +52,7 @@ const INSTANCE_TYPE = requireEnv('INSTANCE_TYPE'); const SUBNET_IDS = requireEnv('SUBNET_IDS').split(','); const INSTANCE_PROFILE_ARN = requireEnv('INSTANCE_PROFILE_ARN'); const WEIGHTS_BUCKET = requireEnv('WEIGHTS_BUCKET'); +const MAX_CONCURRENT_SEEDS = Number(requireEnv('MAX_CONCURRENT_SEEDS')); const REGION = requireEnv('AWS_REGION'); const BOOT_LOG_GROUP = requireEnv('BOOT_LOG_GROUP'); const ENGINE_LOG_GROUP = Object.fromEntries( @@ -206,6 +211,98 @@ async function readDaemonActivity( } } +// How often a caller should re-ask while the weights are still seeding. A +// seed runs for minutes, so nothing is learned by polling faster — and the +// reply stays in the same shape as the other 503s the wake gives out. +const SEED_RETRY_SECONDS = 60; + +/** Instance states that mean a seed's compute is alive. */ +function seedAlive(state: string): boolean { + return state === 'pending' || state === 'running'; +} + +function seedingReply(seedId: string, message: string): LambdaFunctionURLResult { + return jsonResponse(503, { + state: 'seeding', + seedId, + retry_after_seconds: SEED_RETRY_SECONDS, + message, + }); +} + +/** + * The weights gate: a wake must not launch an instance that will sync a + * partial prefix. + * + * Weights are judged by the manifest the seeder writes last — not by the + * silence of an error, and not by the seed's reports alone, which stop + * arriving the moment the seed's process dies. When the weights are absent + * the seed for them is either running, in which case the wake joins it, or + * not, in which case the wake starts it and the caller waits. Either way the + * reply is the same retryable state naming the seed, so the caller's loop can + * keep polling without learning anything about how seeds work. + * + * Returns null when the weights are present and the wake may proceed. + */ +async function seedingGate( + env: string, + config: DeployConfig, +): Promise { + let present: boolean; + try { + present = await weightsPresent(WEIGHTS_BUCKET, config); + } catch (err) { + // A failed manifest read is not "absent": read as absent, a transient + // glitch would pay for a full re-seed; read as present, the wake would + // boot on weights nobody verified. Say the check failed and let the + // caller re-ask, the way deploy answers a seed it cannot start. + console.log(JSON.stringify({ phase: 'seed-check', environment: env, error: errorName(err) })); + return jsonResponse(502, { + error: `could not check whether the weights are present: ${(err as Error).message}`, + }); + } + if (present) { + return null; + } + const seedId = seedIdFor(config.runner, config.modelId, config.quant); + const follow = `follow it with \`spinloop remote seed status ${seedId}\``; + + const inFlight = (await findSeedInstances(TAG_KEY, seedId)).filter((i) => seedAlive(i.state)); + if (inFlight.length > 0) { + console.log( + JSON.stringify({ phase: 'seeding', environment: env, seedId, joined: inFlight[0].instanceId }), + ); + return seedingReply(seedId, `seeding the weights — ${follow}`); + } + + // Counted over alive seeds only: a stopped seed instance holds no compute, + // and it would be a cap that a dead body keeps filled. + const alive = (await findSeedInstances(TAG_KEY)).filter((i) => seedAlive(i.state)); + if (alive.length >= MAX_CONCURRENT_SEEDS) { + console.log(JSON.stringify({ phase: 'seeding-cap', environment: env, seedId, running: alive.length })); + return seedingReply( + seedId, + `${alive.length} seeds are running (cap ${MAX_CONCURRENT_SEEDS}) — waiting for a slot; ${follow}`, + ); + } + + const infra = seedInfraFromEnv(); + try { + const launched = await launchSeedInstance(buildSeedJob(config, infra, ''), infra); + console.log(JSON.stringify({ phase: 'seed-launched', environment: env, seedId, instanceId: launched.instanceId })); + } catch (err) { + // A refused launch is not a reason to fail the start that wanted the model + // served: the next poll retries it, and the idempotency token means a retry + // can never double the compute. + console.log(JSON.stringify({ phase: 'seed-launch', environment: env, seedId, error: errorName(err) })); + return seedingReply( + seedId, + `the seed could not be started (${(err as Error).message}) — retrying; ${follow}`, + ); + } + return seedingReply(seedId, `seeding the weights — ${follow}`); +} + /** POST — launch the environment's instance if needed and block until serving. */ async function wake( env: string, @@ -247,6 +344,13 @@ async function wake( } const baseUrl = baseUrlFor(eip.publicIp, ENGINE_PORT); + // Weights first: a launch against an incomplete prefix would boot the engine + // on it, and a re-wake would keep an old one alive on nothing. + const gate = await seedingGate(env, deployConfig); + if (gate) { + return gate; + } + const existing = await findManagedInstance(TAG_KEY, TAG_VALUE, envFilter(env)); let instanceId: string; let startIssued = false; diff --git a/remote/lib/llm-stack.ts b/remote/lib/llm-stack.ts index 9ed40d7f..99c1081e 100644 --- a/remote/lib/llm-stack.ts +++ b/remote/lib/llm-stack.ts @@ -353,13 +353,17 @@ export class LlmStack extends cdk.Stack { logGroup: lambdaLogGroup('StartFnLogGroup', 'start'), environment: { ...commonEnv, + // Seeding: the weights gate launches the seed itself when the + // weights are not in S3 yet, the same path deploy takes, and reports + // the seed id back so the caller can follow it. + ...seedEnv, + MAX_CONCURRENT_SEEDS: String(cfg.maxConcurrentSeeds), AMI_ROLE_TAG_KEY, AMI_ROLE_TAG_VALUE, AMI_RUNNER_TAG_KEY, INSTANCE_TYPE: cfg.instanceType, SUBNET_IDS: vpc.publicSubnets.map((s) => s.subnetId).join(','), INSTANCE_PROFILE_ARN: instanceProfile.instanceProfileArn, - WEIGHTS_BUCKET: weightsBucket.bucketName, // Log groups the instance's CloudWatch agent ships to, one env var // per runner by convention (logGroupEnvVar). The group is fixed; the // stream (/) is filled in at boot. @@ -414,6 +418,12 @@ export class LlmStack extends cdk.Stack { }), ); startFn.addToRolePolicy(describeStatement); + // The weights gate: launching the seed instance needs exactly the grants + // the deploy and seed Lambdas have — run it, tag it, pass the seed role, + // read the stock image id. + seedLaunchStatements().forEach((s) => startFn.addToRolePolicy(s)); + // Read-only on the weights: only to check whether the manifest is there. + weightsBucket.grantRead(startFn, 'models/*'); sendCommandStatements().forEach((s) => startFn.addToRolePolicy(s)); // Find the latest baked AMI, the environment's EIP and its security group. // These Describe calls have no resource-level scoping. diff --git a/remote/test/seed-discovery.test.ts b/remote/test/seed-discovery.test.ts new file mode 100644 index 00000000..aee69b5a --- /dev/null +++ b/remote/test/seed-discovery.test.ts @@ -0,0 +1,41 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +// The discovery contract the seed and start Lambdas both build on: one filter, +// so the two agree on which instance is a seed's compute. The gate's own tests +// stub findManagedInstances with this shape already assumed, so this is the one +// test that checks the wrapper builds the filter it claims to. + +const findManagedInstances = vi.fn(); + +vi.mock('../lambda/shared/aws', async (importOriginal) => ({ + ...(await importOriginal()), + findManagedInstances: (...args: unknown[]) => findManagedInstances(...args), +})); + +let findSeedInstances: typeof import('../lambda/shared/seed/discovery').findSeedInstances; +let SEED_ID_TAG_KEY: string; +let SEED_TAG_VALUE: string; + +beforeAll(async () => { + ({ findSeedInstances } = await import('../lambda/shared/seed/discovery')); + ({ SEED_ID_TAG_KEY, SEED_TAG_VALUE } = await import('../lambda/shared/seed/identity')); +}); + +describe('finding seed instances', () => { + beforeEach(() => { + vi.clearAllMocks(); + findManagedInstances.mockResolvedValue([]); + }); + + it('selects on the seed tag value and nothing else when no id is given', async () => { + await findSeedInstances('cloud-vm-llm'); + expect(findManagedInstances).toHaveBeenCalledWith('cloud-vm-llm', SEED_TAG_VALUE, []); + }); + + it('narrows to the seed id when one is given', async () => { + await findSeedInstances('cloud-vm-llm', 'vllm--org-model--Q4_K_M'); + expect(findManagedInstances).toHaveBeenCalledWith('cloud-vm-llm', SEED_TAG_VALUE, [ + { Name: `tag:${SEED_ID_TAG_KEY}`, Values: ['vllm--org-model--Q4_K_M'] }, + ]); + }); +}); diff --git a/remote/test/stack.test.ts b/remote/test/stack.test.ts index 81e39a9c..1cd24bd1 100644 --- a/remote/test/stack.test.ts +++ b/remote/test/stack.test.ts @@ -226,6 +226,51 @@ describe('LlmStack (control plane)', () => { expect(actions).toContain('ec2:DescribeSecurityGroups'); }); + it('carries the seed environment and launch grants on the start Lambda', () => { + // The weights gate launches the seed itself, so the start Lambda needs + // the seed material deploy and seed carry — the actions test above cannot + // see this, because those grants duplicate the start's own. + const fns = template.findResources('AWS::Lambda::Function'); + const start = Object.values(fns).find((f) => + String(f.Properties.Description).includes('Launches an environment instance'), + ); + const env = start!.Properties.Environment.Variables; + expect(env.MAX_CONCURRENT_SEEDS).toBeDefined(); + expect(env.SEED_INSTANCE_TYPE).toBeDefined(); + expect(env.SEED_INSTANCE_PROFILE_ARN).toBeDefined(); + expect(env.SEEDER_BUCKET).toBeDefined(); + + const policies = template.findResources('AWS::IAM::Policy'); + const startPolicy = Object.values(policies).find((p) => + String(p.Properties.PolicyName).startsWith('StartFnServiceRoleDefaultPolicy'), + ); + expect(startPolicy).toBeDefined(); + const statements = startPolicy!.Properties.PolicyDocument.Statement as { + Action: string | string[]; + Resource?: unknown; + Condition?: unknown; + }[]; + // The start's own PassRole covers the inference role, so the seed one is + // found by its resource, not by the action. + const passSeedRole = statements.find( + (s) => [s.Action].flat().includes('iam:PassRole') && JSON.stringify(s.Resource).includes('SeedRole'), + ); + expect(passSeedRole).toBeDefined(); + expect(JSON.stringify(passSeedRole!.Condition)).toContain('ec2.amazonaws.com'); + const amiParameter = statements.find( + (s) => + [s.Action].flat().includes('ssm:GetParameter') && + JSON.stringify(s.Resource).includes('ami-amazon-linux-latest'), + ); + expect(amiParameter).toBeDefined(); + const weightsRead = statements.find( + (s) => + [s.Action].flat().some((a) => a.startsWith('s3:GetObject')) && + JSON.stringify(s.Resource).includes('/models/*'), + ); + expect(weightsRead).toBeDefined(); + }); + it('scopes per-environment SSM and secret access to the cloud-vm-llm prefix', () => { const statements = allPolicyStatements(template); const ssmStatement = statements.find((s) => [s.Action].flat().includes('ssm:PutParameter')); diff --git a/remote/test/start-boot-failure.test.ts b/remote/test/start-boot-failure.test.ts index d00d0754..890eb0ba 100644 --- a/remote/test/start-boot-failure.test.ts +++ b/remote/test/start-boot-failure.test.ts @@ -19,6 +19,7 @@ const LAMBDA_ENV = { SUBNET_IDS: 'subnet-test', INSTANCE_PROFILE_ARN: 'arn:aws:iam::0:instance-profile/test', WEIGHTS_BUCKET: 'test-bucket', + MAX_CONCURRENT_SEEDS: '2', AWS_REGION: 'us-east-1', BOOT_LOG_GROUP: '/test/boot', LLAMACPP_LOG_GROUP: '/test/llamacpp', @@ -54,6 +55,12 @@ vi.mock('../lambda/shared/environments', async (importOriginal) => ({ readEnvApiKey: (...args: unknown[]) => readEnvApiKey(...args), })); +vi.mock('../lambda/shared/seed', () => ({ + // The weights gate asks this on every wake; these tests are about the wake + // itself, so the weights are always present. + weightsPresent: async () => true, +})); + let handler: (event: LambdaFunctionURLEvent, context: Context) => Promise; let buildInferenceUserData: (env: string, cfg: DeployConfig) => string; let BOOT_FAILED_MARKER: string; diff --git a/remote/test/start-launch.test.ts b/remote/test/start-launch.test.ts index 93cddae2..bc319a18 100644 --- a/remote/test/start-launch.test.ts +++ b/remote/test/start-launch.test.ts @@ -18,6 +18,7 @@ const LAMBDA_ENV = { SUBNET_IDS: 'subnet-test', INSTANCE_PROFILE_ARN: 'arn:aws:iam::0:instance-profile/test', WEIGHTS_BUCKET: 'test-bucket', + MAX_CONCURRENT_SEEDS: '2', AWS_REGION: 'us-east-1', BOOT_LOG_GROUP: '/test/boot', LLAMACPP_LOG_GROUP: '/test/llamacpp', @@ -59,6 +60,12 @@ vi.mock('../lambda/shared/environments', async (importOriginal) => ({ readEnvApiKey: (...args: unknown[]) => readEnvApiKey(...args), })); +vi.mock('../lambda/shared/seed', () => ({ + // The weights gate asks this on every wake; these tests are about the wake + // itself, so the weights are always present. + weightsPresent: async () => true, +})); + let handler: (event: LambdaFunctionURLEvent, context: Context) => Promise; beforeAll(async () => { diff --git a/remote/test/start-rewake.test.ts b/remote/test/start-rewake.test.ts index ca20e97c..5dcbd005 100644 --- a/remote/test/start-rewake.test.ts +++ b/remote/test/start-rewake.test.ts @@ -17,6 +17,7 @@ const LAMBDA_ENV = { SUBNET_IDS: 'subnet-test', INSTANCE_PROFILE_ARN: 'arn:aws:iam::0:instance-profile/test', WEIGHTS_BUCKET: 'test-bucket', + MAX_CONCURRENT_SEEDS: '2', AWS_REGION: 'us-east-1', BOOT_LOG_GROUP: '/test/boot', LLAMACPP_LOG_GROUP: '/test/llamacpp', @@ -60,6 +61,12 @@ vi.mock('../lambda/shared/environments', async (importOriginal) => ({ readEnvApiKey: (...args: unknown[]) => readEnvApiKey(...args), })); +vi.mock('../lambda/shared/seed', () => ({ + // The weights gate asks this on every wake; these tests are about the wake + // itself, so the weights are always present. + weightsPresent: async () => true, +})); + let handler: (event: LambdaFunctionURLEvent, context: Context) => Promise; beforeAll(async () => { diff --git a/remote/test/start-seeding.test.ts b/remote/test/start-seeding.test.ts new file mode 100644 index 00000000..df205355 --- /dev/null +++ b/remote/test/start-seeding.test.ts @@ -0,0 +1,307 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Context, LambdaFunctionURLEvent, LambdaFunctionURLResult } from 'aws-lambda'; +import type { InstanceInfo } from '../lambda/shared/aws'; +import { DAEMON_STATUS_CMD } from '../lambda/shared/daemon'; + +// The weights gate of the start Lambda: a wake must not launch an instance +// whose weights are still seeding, and a wake with absent weights starts the +// seed rather than booting on a partial prefix. All AWS calls are stubbed. + +const LAMBDA_ENV = { + TAG_KEY: 'cloud-vm-llm:managed', + TAG_VALUE: 'true', + ENGINE_PORT: '8000', + AMI_ROLE_TAG_KEY: 'cloud-vm-llm:role', + AMI_ROLE_TAG_VALUE: 'runtime-ami', + AMI_RUNNER_TAG_KEY: 'cloud-vm-llm:runner', + INSTANCE_TYPE: 'g6e.xlarge', + SUBNET_IDS: 'subnet-test', + INSTANCE_PROFILE_ARN: 'arn:aws:iam::0:instance-profile/test', + WEIGHTS_BUCKET: 'test-bucket', + MAX_CONCURRENT_SEEDS: '2', + AWS_REGION: 'us-east-1', + BOOT_LOG_GROUP: '/test/boot', + LLAMACPP_LOG_GROUP: '/test/llamacpp', + VLLM_LOG_GROUP: '/test/vllm', +}; + +const findManagedInstance = vi.fn(); +const findManagedInstances = vi.fn(); +const getInstance = vi.fn(); +const startEngineDaemon = vi.fn(); +const runInstance = vi.fn(); +const findLatestAmi = vi.fn(); +const tagInstance = vi.fn(); +const associateEip = vi.fn(); +const isSsmAgentOnline = vi.fn(); +const runShellCommand = vi.fn(); +const readDeployConfig = vi.fn(); +const findEnvEip = vi.fn(); +const findEnvSecurityGroup = vi.fn(); +const readEnvApiKey = vi.fn(); +const weightsPresent = vi.fn(); +const launchSeedInstance = vi.fn(); +const buildSeedJob = vi.fn(); +const startInstance = vi.fn(); + +vi.mock('../lambda/shared/aws', async (importOriginal) => ({ + ...(await importOriginal()), + findManagedInstance: (...args: unknown[]) => findManagedInstance(...args), + findManagedInstances: (...args: unknown[]) => findManagedInstances(...args), + getInstance: (...args: unknown[]) => getInstance(...args), + startInstance: (...args: unknown[]) => startInstance(...args), + startEngineDaemon: (...args: unknown[]) => startEngineDaemon(...args), + runInstance: (...args: unknown[]) => runInstance(...args), + findLatestAmi: (...args: unknown[]) => findLatestAmi(...args), + tagInstance: (...args: unknown[]) => tagInstance(...args), + associateEip: (...args: unknown[]) => associateEip(...args), + isSsmAgentOnline: (...args: unknown[]) => isSsmAgentOnline(...args), + runShellCommand: (...args: unknown[]) => runShellCommand(...args), + readDeployConfig: (...args: unknown[]) => readDeployConfig(...args), +})); + +vi.mock('../lambda/shared/environments', async (importOriginal) => ({ + ...(await importOriginal()), + findEnvEip: (...args: unknown[]) => findEnvEip(...args), + findEnvSecurityGroup: (...args: unknown[]) => findEnvSecurityGroup(...args), + readEnvApiKey: (...args: unknown[]) => readEnvApiKey(...args), +})); + +vi.mock('../lambda/shared/seed', () => ({ + weightsPresent: (...args: unknown[]) => weightsPresent(...args), +})); + +vi.mock('../lambda/shared/seed/launch', () => ({ + seedInfraFromEnv: () => ({ bucket: 'test-bucket' }), + buildSeedJob: (...args: unknown[]) => buildSeedJob(...args), + launchSeedInstance: (...args: unknown[]) => launchSeedInstance(...args), +})); + +let handler: (event: LambdaFunctionURLEvent, context: Context) => Promise; + +beforeAll(async () => { + Object.assign(process.env, LAMBDA_ENV); + ({ handler } = await import('../lambda/start/index')); +}); + +const wakeEvent = { + queryStringParameters: { env: 'dev' }, + requestContext: { http: { method: 'POST' } }, +} as unknown as LambdaFunctionURLEvent; + +const context = { getRemainingTimeInMillis: () => 600_000 } as unknown as Context; + +function structured(result: LambdaFunctionURLResult): { statusCode: number; body: string } { + return result as { statusCode: number; body: string }; +} + +const CONFIG = { + runner: 'llamacpp', + modelId: 'org/model', + quant: 'Q4_K_M', + weightsPrefix: 'llamacpp/org/model/Q4_K_M', + contextSize: 32768, + servedModelName: 'friendly', + serveArgs: [], + companions: {}, +}; + +// seedIdFor('llamacpp', 'org/model', 'Q4_K_M'). +const SEED_ID = 'llamacpp--org-model--Q4_K_M'; + +/** A seed instance carrying its own id tag. */ +function seedInstance(id: string, state: string, seedId: string = SEED_ID): InstanceInfo { + return { instanceId: id, state, tags: { 'cloud-vm-llm:seed-id': seedId } }; +} + +// The seed instances DescribeInstances would return, and which of them answer +// the gate's questions. The filter check mirrors findSeedInstances: a seed-id +// tag filter narrows, and no filter means every seed instance. +let seeds: InstanceInfo[] = []; +function seedLookup( + _key: string, + _value: string, + filters: { Name: string; Values: string[] }[] = [], +): InstanceInfo[] { + const idFilter = filters.find((f) => f.Name === 'tag:cloud-vm-llm:seed-id'); + return idFilter ? seeds.filter((i) => i.tags?.['cloud-vm-llm:seed-id'] === idFilter.Values[0]) : seeds; +} + +beforeEach(() => { + vi.clearAllMocks(); + seeds = []; + // A parsed config, whole: the boot script iterates it and the start's body + // renders it, on the path where the weights are present. + readDeployConfig.mockResolvedValue(CONFIG); + weightsPresent.mockResolvedValue(true); + findSeedMock(); + findEnvEip.mockResolvedValue({ publicIp: '198.51.100.7', allocationId: 'eipalloc-test' }); + findEnvSecurityGroup.mockResolvedValue('sg-test'); + readEnvApiKey.mockResolvedValue('sk-test'); + isSsmAgentOnline.mockResolvedValue(true); + runShellCommand.mockImplementation((_instanceId: string, command: string) => + command === DAEMON_STATUS_CMD + ? Promise.resolve({ status: 'Success', stdout: JSON.stringify({ state: 'stopped' }) }) + : Promise.resolve({ status: 'Success', stdout: '200' }), + ); + startInstance.mockResolvedValue(undefined); + startEngineDaemon.mockResolvedValue(true); + findManagedInstance.mockResolvedValue(null); + getInstance.mockResolvedValue({ instanceId: 'i-new', state: 'running', launchTime: new Date() }); + runInstance.mockResolvedValue('i-new'); + findLatestAmi.mockResolvedValue({ imageId: 'ami-test1', rootVolumeSizeGb: 80 }); + launchSeedInstance.mockResolvedValue({ seedId: SEED_ID, instanceId: 'i-seed', started: true }); + buildSeedJob.mockImplementation( + (cfg: unknown) => ({ seedId: SEED_ID, cfg }), + ); +}); + +function findSeedMock() { + findManagedInstances.mockImplementation((_k: string, _v: string, f?: { Name: string; Values: string[] }[]) => + Promise.resolve(seedLookup(_k, _v, f ?? [])), + ); +} + +describe('the weights gate', () => { + it('proceeds to launch when the weights are present', async () => { + const result = await handler(wakeEvent, context); + + expect(structured(result).statusCode).toBe(200); + expect(runInstance).toHaveBeenCalled(); + expect(launchSeedInstance).not.toHaveBeenCalled(); + }); + + it('joins a running seed without launching an instance', async () => { + weightsPresent.mockResolvedValue(false); + seeds = [seedInstance('i-seed', 'running')]; + + const result = await handler(wakeEvent, context); + + const reply = JSON.parse(structured(result).body); + expect(structured(result).statusCode).toBe(503); + expect(reply.state).toBe('seeding'); + expect(reply.seedId).toBe(SEED_ID); + expect(reply.retry_after_seconds).toBe(60); + expect(reply.message).toContain(`spinloop remote seed status ${SEED_ID}`); + expect(runInstance).not.toHaveBeenCalled(); + expect(launchSeedInstance).not.toHaveBeenCalled(); + }); + + it('reports a manifest read failure as a 502, not a retryable seeding state', async () => { + // Read the failure as absent and a transient glitch pays for a full + // re-seed; read it as present and the wake boots on unverified weights. + // The gate says the check failed instead, and stops there. + weightsPresent.mockRejectedValue(new Error('AccessDenied')); + seeds = [seedInstance('i-seed', 'running')]; + const result = await handler(wakeEvent, context); + const body = JSON.parse(structured(result).body); + expect(structured(result).statusCode).toBe(502); + expect(body.state).toBeUndefined(); + expect(body.error).toContain('could not check whether the weights are present'); + expect(body.error).toContain('AccessDenied'); + expect(findManagedInstances).not.toHaveBeenCalled(); + expect(launchSeedInstance).not.toHaveBeenCalled(); + expect(runInstance).not.toHaveBeenCalled(); + }); + + it('holds the re-wake too, not just the launch', async () => { + weightsPresent.mockResolvedValue(false); + seeds = [seedInstance('i-seed', 'running')]; + // The environment's instance exists and is stopped: absent the gate this + // is the re-wake path. + findManagedInstance.mockResolvedValue({ instanceId: 'i-old', state: 'stopped' }); + const result = await handler(wakeEvent, context); + const reply = JSON.parse(structured(result).body); + expect(structured(result).statusCode).toBe(503); + expect(reply.state).toBe('seeding'); + expect(reply.seedId).toBe(SEED_ID); + // The gate answers before the re-wake: the stopped instance is not + // started and no fresh one is launched. + expect(startInstance).not.toHaveBeenCalled(); + expect(runInstance).not.toHaveBeenCalled(); + }); + + it('counts a pending seed as in flight', async () => { + weightsPresent.mockResolvedValue(false); + seeds = [seedInstance('i-seed', 'pending')]; + + const result = await handler(wakeEvent, context); + + expect(JSON.parse(structured(result).body).state).toBe('seeding'); + expect(launchSeedInstance).not.toHaveBeenCalled(); + expect(runInstance).not.toHaveBeenCalled(); + }); + + it('starts a fresh seed when a stopped one is all that remains', async () => { + // A stopped seed instance is a dead seed: its joined state is already + // failed, and waiting on it would wedge every start for these weights. + weightsPresent.mockResolvedValue(false); + seeds = [seedInstance('i-seed', 'stopped')]; + + const result = await handler(wakeEvent, context); + + const reply = JSON.parse(structured(result).body); + expect(structured(result).statusCode).toBe(503); + expect(reply.state).toBe('seeding'); + expect(reply.seedId).toBe(SEED_ID); + expect(launchSeedInstance).toHaveBeenCalledTimes(1); + expect(runInstance).not.toHaveBeenCalled(); + }); + + it('starts the seed from the environment deploy-config, companions and all', async () => { + weightsPresent.mockResolvedValue(false); + const withDrafter = { ...CONFIG, companions: { draft: 'dflash-kquant.gguf' } }; + readDeployConfig.mockResolvedValue(withDrafter); + + const result = await handler(wakeEvent, context); + + expect(JSON.parse(structured(result).body).state).toBe('seeding'); + expect(buildSeedJob).toHaveBeenCalledWith(withDrafter, expect.anything(), ''); + expect(launchSeedInstance).toHaveBeenCalledTimes(1); + }); + + it('waits for a slot rather than exceeding the seed cap', async () => { + weightsPresent.mockResolvedValue(false); + seeds = [ + seedInstance('i-a', 'running', 'llamacpp--other-a'), + seedInstance('i-b', 'running', 'llamacpp--other-b'), + ]; + + const result = await handler(wakeEvent, context); + + const reply = JSON.parse(structured(result).body); + expect(structured(result).statusCode).toBe(503); + expect(reply.state).toBe('seeding'); + expect(reply.seedId).toBe(SEED_ID); + expect(reply.message).toMatch(/2 seeds are running \(cap 2\)/); + expect(launchSeedInstance).not.toHaveBeenCalled(); + expect(runInstance).not.toHaveBeenCalled(); + }); + + it('does not count stopped seeds against the cap', async () => { + weightsPresent.mockResolvedValue(false); + seeds = [ + seedInstance('i-a', 'stopped', 'llamacpp--other-a'), + seedInstance('i-b', 'running', 'llamacpp--other-b'), + ]; + + await handler(wakeEvent, context); + + expect(launchSeedInstance).toHaveBeenCalledTimes(1); + }); + + it('reports a launch failure as retryable seeding, not a fatal error', async () => { + weightsPresent.mockResolvedValue(false); + launchSeedInstance.mockRejectedValue(new Error('no capacity in the seed zone')); + + const result = await handler(wakeEvent, context); + + const reply = JSON.parse(structured(result).body); + expect(structured(result).statusCode).toBe(503); + expect(reply.state).toBe('seeding'); + expect(reply.seedId).toBe(SEED_ID); + expect(reply.message).toContain('no capacity in the seed zone'); + expect(reply.message).toContain(`spinloop remote seed status ${SEED_ID}`); + expect(runInstance).not.toHaveBeenCalled(); + }); +}); diff --git a/remote/test/start-status.test.ts b/remote/test/start-status.test.ts index 3f8f527b..56aeed8b 100644 --- a/remote/test/start-status.test.ts +++ b/remote/test/start-status.test.ts @@ -19,6 +19,7 @@ const LAMBDA_ENV = { SUBNET_IDS: 'subnet-test', INSTANCE_PROFILE_ARN: 'arn:aws:iam::0:instance-profile/test', WEIGHTS_BUCKET: 'test-bucket', + MAX_CONCURRENT_SEEDS: '2', AWS_REGION: 'us-east-1', BOOT_LOG_GROUP: '/test/boot', LLAMACPP_LOG_GROUP: '/test/llamacpp', @@ -42,6 +43,12 @@ vi.mock('../lambda/shared/environments', async (importOriginal) => ({ findEnvEip: (...args: unknown[]) => findEnvEip(...args), })); +vi.mock('../lambda/shared/seed', () => ({ + // The weights gate asks this on every wake; these tests are about the wake + // itself, so the weights are always present. + weightsPresent: async () => true, +})); + let handler: (event: LambdaFunctionURLEvent, context: Context) => Promise; beforeAll(async () => { diff --git a/remote/test/start.test.ts b/remote/test/start.test.ts index 599b91d1..31e50663 100644 --- a/remote/test/start.test.ts +++ b/remote/test/start.test.ts @@ -17,6 +17,7 @@ const LAMBDA_ENV = { // guard's 12-digit patterns can never mistake for a real ARN. INSTANCE_PROFILE_ARN: 'arn:aws:iam::0:instance-profile/test', WEIGHTS_BUCKET: 'test-bucket', + MAX_CONCURRENT_SEEDS: '2', AWS_REGION: 'us-east-1', BOOT_LOG_GROUP: '/test/boot', LLAMACPP_LOG_GROUP: '/test/llamacpp', diff --git a/remote/vitest.config.ts b/remote/vitest.config.ts index ad957b7c..d96cb194 100644 --- a/remote/vitest.config.ts +++ b/remote/vitest.config.ts @@ -6,7 +6,9 @@ export default defineConfig({ // trees run in one `pnpm test` so there is a single lane to keep green. include: ['test/**/*.test.ts', 'seeder/test/**/*.test.ts'], environment: 'node', - // Stack synth (with esbuild bundling of the Lambdas) is slow on first run. + // Stack synth (with esbuild bundling of the Lambdas) is slow on first run + // and happens in a beforeAll hook, so the hook needs the same headroom. testTimeout: 120_000, + hookTimeout: 120_000, }, });