fix(remote): guard start against launching while the weights are seeding

internal/fleet/start_phase.go

@@ -34,6 +34,10 @@ const (
 	// PhaseReconnecting: the attempt's connection dropped, and the next one is
 	// due at RetryAt.
 	PhaseReconnecting
+	// PhaseSeeding: the weights are still being fetched, so no instance has
+	// been launched yet and there is no boot to time — the count-up answers
+	// "how long has the fetch been going".
+	PhaseSeeding
 )
 
 // StartPhase is a start's current situation. It carries no rendered text and no
@@ -57,6 +61,7 @@ type StartPhase struct {
 const (
 	stateNoCapacity = "no-capacity"
 	stateReady      = "ready"
+	stateSeeding    = "seeding"
 )
 
 // RenderPhase is the phase's line at time now. A wait counts down towards
@@ -72,6 +77,11 @@ func RenderPhase(p StartPhase, now time.Time) string {
 			return "booting"
 		}
 		return "booting (" + formatPhaseDuration(now.Sub(p.Since)) + ")"
+	case PhaseSeeding:
+		if p.Since.IsZero() {
+			return "seeding the weights"
+		}
+		return "seeding the weights (" + formatPhaseDuration(now.Sub(p.Since)) + ")"
 	case PhaseReconnecting:
 		line := "connection dropped"
 		if p.Detail != "" {
@@ -163,8 +173,10 @@ func (t *startPhases) state(s string) {
 		// connection: each described the attempt before it. It does not
 		// supersede a boot — once a reply has reported the instance coming
 		// up, the attempts that follow are polls of that same boot, and its
-		// elapsed time counts from the first reply that reported it.
-		if t.phase.Kind != PhaseBooting || !t.begun {
+		// elapsed time counts from the first reply that reported it. A
+		// seeding wait is the same shape: the attempts that follow a seeding
+		// reply poll the same fetch, and its clock keeps running.
+		if t.phase.Kind != PhaseBooting && t.phase.Kind != PhaseSeeding || !t.begun {
 			t.enter(PhaseAttempting, "")
 		}
 	case s == stateReady:
@@ -176,6 +188,11 @@ func (t *startPhases) state(s string) {
 		// line remote.Start writes next, so the wait carries no due time
 		// until that line arrives.
 		t.enter(PhaseWaitingCapacity, s)
+	case s == stateSeeding:
+		// No instance exists yet, so this is not a boot: the fetch is what
+		// the count-up measures. enter() keeps Since on the first seeding
+		// reply, so repeated polls do not restart the clock.
+		t.enter(PhaseSeeding, s)
 	default:
 		t.enter(PhaseBooting, s)
 	}

internal/fleet/start_phase_test.go

@@ -54,6 +54,54 @@ func TestRenderPhaseIsComputedAtDrawTime(t *testing.T) {
 	}
 }
 
+// A seeding reply is not a boot: no instance exists yet, and the line counts
+// up from the first seeding reply. The attempts that follow it poll the same
+// fetch, so they must not restart the clock.
+func TestStartPhasesSeeding(t *testing.T) {
+	base := time.Date(2026, 9, 3, 12, 0, 0, 0, time.UTC)
+
+	var got []StartPhase
+	progress, onState := StartPhases(func(p StartPhase) { got = append(got, p) })
+
+	onState(remote.StateInFlight)
+	onState(stateSeeding)
+	progress("seeding the weights (seed llamacpp--org-model--Q4_K_M); retrying in 60s")
+	// The next attempt supersedes nothing: it polls the fetch the seeding
+	// reply reported, and the reply that follows reports the same fetch.
+	onState(remote.StateInFlight)
+	onState(stateSeeding)
+
+	want := []StartPhaseKind{PhaseAttempting, PhaseSeeding}
+	kinds := make([]StartPhaseKind, len(got))
+	for i, p := range got {
+		kinds[i] = p.Kind
+	}
+	if len(kinds) != len(want) {
+		t.Fatalf("phases = %v, want %v", kinds, want)
+	}
+	for i := range want {
+		if kinds[i] != want[i] {
+			t.Fatalf("phases = %v, want %v", kinds, want)
+		}
+	}
+	if got[1].Detail != stateSeeding {
+		t.Errorf("seeding phase detail = %q, want %q", got[1].Detail, stateSeeding)
+	}
+
+	// The rendered line counts up from the first seeding reply at two times.
+	seed := StartPhase{Kind: PhaseSeeding, Since: base, Detail: stateSeeding}
+	if line := RenderPhase(seed, base.Add(30*time.Second)); line != "seeding the weights (30s)" {
+		t.Errorf("RenderPhase(seed, +30s) = %q", line)
+	}
+	if line := RenderPhase(seed, base.Add(13*time.Minute)); line != "seeding the weights (13m 0s)" {
+		t.Errorf("RenderPhase(seed, +13m) = %q", line)
+	}
+	// A phase drawn without a clock renders its name alone, as a boot does.
+	if line := RenderPhase(StartPhase{Kind: PhaseSeeding, Detail: stateSeeding}, base); line != "seeding the weights" {
+		t.Errorf("RenderPhase(seed, no Since) = %q", line)
+	}
+}
+
 // The mapping from remote.Start's two callbacks onto phases: an attempt goes
 // out, is refused for capacity with a due time for the next one, and the
 // attempt that follows retires the refusal rather than leaving it standing

internal/remote/remote.go

@@ -326,6 +326,23 @@ var startRetryWait = 5 * time.Second
 // underway, not a capacity wait.
 const StateInFlight = "in-flight"
 
+// stateSeeding is the control plane's word for "the weights are still being
+// fetched" — no instance has been launched yet, so there is no boot to wait
+// on, and the reply's seedId is what an operator follows to see how far the
+// fetch has got.
+const stateSeeding = "seeding"
+
+// giveUpWaiting is Start's error when the caller's deadline expires mid-wait.
+// When the last reply was the seeding state, the useful next steps are
+// following the seed and resuming the wait, so the error carries both.
+func giveUpWaiting(ctx context.Context, state, seedID string) error {
+	if state == stateSeeding && seedID != "" {
+		return fmt.Errorf("gave up waiting for the endpoint: the weights are still seeding (seed %s) — follow it with `spinloop remote seed status %s`, and re-run start with a longer --timeout: %w",
+			seedID, seedID, ctx.Err())
+	}
+	return fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err())
+}
+
 // Start boots the instance and blocks until the model is serving, retrying
 // while the endpoint reports it is still starting. progress is called with a
 // status line before each wait. onState, when non-nil, is called with the raw
@@ -354,6 +371,11 @@ func Start(ctx context.Context, cfg Config, progress func(string), onState func(
 			startURL = u.String()
 		}
 	}
+	// The last reply's state and seed id, for the give-up error: a deadline
+	// that expires mid-seed is not the same situation as one that expires
+	// mid-boot, and the operator should be told which.
+	lastState := ""
+	lastSeedID := ""
 	for {
 		// Supersedes whatever the previous attempt reported — including a
 		// no-capacity reply: this attempt has not refused anything yet, and a
@@ -369,7 +391,7 @@ func Start(ctx context.Context, cfg Config, progress func(string), onState func(
 				progress(fmt.Sprintf("connection dropped (%v); retrying in %s", urlErr.Unwrap(), startRetryWait))
 				select {
 				case <-ctx.Done():
-					return nil, fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err())
+					return nil, giveUpWaiting(ctx, lastState, lastSeedID)
 				case <-time.After(startRetryWait):
 				}
 				continue
@@ -383,14 +405,21 @@ func Start(ctx context.Context, cfg Config, progress func(string), onState func(
 		case resp.StatusCode == http.StatusOK && resp.State == "ready":
 			return resp, nil
 		case resp.StatusCode == http.StatusServiceUnavailable:
+			lastState, lastSeedID = resp.State, resp.SeedID
 			wait := resp.RetryAfterSeconds
 			if wait <= 0 {
 				wait = 1
 			}
-			progress(fmt.Sprintf("instance %s; retrying in %ds", resp.State, wait))
+			// "instance <state>" would be a lie while the weights seed: there
+			// is no instance yet, and the seed is what the operator follows.
+			if resp.State == stateSeeding && resp.SeedID != "" {
+				progress(fmt.Sprintf("seeding the weights (seed %s); retrying in %ds", resp.SeedID, wait))
+			} else {
+				progress(fmt.Sprintf("instance %s; retrying in %ds", resp.State, wait))
+			}
 			select {
 			case <-ctx.Done():
-				return nil, fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err())
+				return nil, giveUpWaiting(ctx, lastState, lastSeedID)
 			case <-time.After(time.Duration(wait) * time.Second):
 			}
 		default:

internal/remote/remote_test.go

@@ -192,6 +192,131 @@ func TestStart_RetriesUntilReady(t *testing.T) {
 	}
 }
 
+// A 503 seeding reply names the seed on every progress line — there is no
+// instance to speak of yet — and the loop keeps polling until the weights
+// are in and the model is serving.
+func TestStart_NamesTheSeedWhileSeeding(t *testing.T) {
+	stubAWSEnv(t)
+	calls := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		calls++
+		w.Header().Set("Content-Type", "application/json")
+		if calls < 3 {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			w.Write([]byte(`{"state":"seeding","seedId":"llamacpp--org-model--Q4_K_M","retry_after_seconds":0}`))
+			return
+		}
+		w.Write([]byte(`{"state":"ready","base_url":"http://198.51.100.1:8000/v1","api_key":"sk-test"}`))
+	}))
+	defer server.Close()
+
+	cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"}
+	var progress []string
+	resp, err := Start(context.Background(), cfg, func(msg string) { progress = append(progress, msg) }, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.State != "ready" {
+		t.Errorf("unexpected response: %+v", resp)
+	}
+	if calls != 3 {
+		t.Errorf("expected 3 calls, got %d", calls)
+	}
+	for _, line := range progress {
+		if !strings.Contains(line, "seeding the weights") || !strings.Contains(line, "llamacpp--org-model--Q4_K_M") {
+			t.Errorf("progress line does not name the seed: %q", line)
+		}
+	}
+}
+
+// A deadline that expires mid-seed is not the generic give-up: the weights
+// are still being fetched, and the error carries the seed's follow command
+// and the longer-timeout hint.
+func TestStart_GiveUpDuringSeedingNamesTheSeed(t *testing.T) {
+	stubAWSEnv(t)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		w.Write([]byte(`{"state":"seeding","seedId":"llamacpp--org-model--Q4_K_M","retry_after_seconds":30}`))
+	}))
+	defer server.Close()
+
+	cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"}
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	_, err := Start(ctx, cfg, func(string) {}, nil, nil)
+	if err == nil {
+		t.Fatal("expected an error when the deadline expires")
+	}
+	for _, want := range []string{"seeding", "llamacpp--org-model--Q4_K_M", "spinloop remote seed status", "--timeout"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("give-up error does not name %q: %v", want, err)
+		}
+	}
+}
+
+// The last reply was seeding and then the connection dropped: the give-up
+// that follows the drop still names the seed, so the state a reply carried
+// survives a dropped connection.
+func TestStart_GiveUpAfterADroppedConnectionNamesTheSeed(t *testing.T) {
+	stubAWSEnv(t)
+	origWait := startRetryWait
+	startRetryWait = time.Second
+	t.Cleanup(func() { startRetryWait = origWait })
+
+	calls := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		calls++
+		if calls == 1 {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusServiceUnavailable)
+			w.Write([]byte(`{"state":"seeding","seedId":"llamacpp--org-model--Q4_K_M","retry_after_seconds":0}`))
+			return
+		}
+		conn, _, err := w.(http.Hijacker).Hijack()
+		if err != nil {
+			t.Fatal(err)
+		}
+		conn.Close()
+	}))
+	defer server.Close()
+
+	cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"}
+	ctx, cancel := context.WithTimeout(context.Background(), 1800*time.Millisecond)
+	defer cancel()
+	_, err := Start(ctx, cfg, func(string) {}, nil, nil)
+	if err == nil {
+		t.Fatal("expected an error once the deadline passed")
+	}
+	for _, want := range []string{"seeding", "llamacpp--org-model--Q4_K_M", "spinloop remote seed status"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("give-up error does not name %q: %v", want, err)
+		}
+	}
+}
+
+// A deadline that expires outside seeding keeps today's give-up message.
+func TestStart_GiveUpOutsideSeedingIsGeneric(t *testing.T) {
+	stubAWSEnv(t)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		w.Write([]byte(`{"state":"starting","retry_after_seconds":30}`))
+	}))
+	defer server.Close()
+
+	cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"}
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	_, err := Start(ctx, cfg, func(string) {}, nil, nil)
+	if err == nil {
+		t.Fatal("expected an error when the deadline expires")
+	}
+	if want := "gave up waiting for the endpoint: context deadline exceeded"; err.Error() != want {
+		t.Errorf("give-up error = %q, want %q", err.Error(), want)
+	}
+}
+
 // onState must see both the raw state of every poll and each attempt as it is
 // issued, so a caller can tell a capacity wait apart from a boot rather than
 // assume the instance is starting.

openspec/changes/archive/2026-09-06-guard-start-during-seed/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-09-06