Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cmd/spinloop/dashboard_detail.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,11 +183,12 @@ func (m *dashModel) detailSectionHeights() (metrics, log int) {
}

// detailNodeLines is the metrics section: the same lines the node's tile draws,
// for the node the view is open on.
// for the node the view is open on, in the board's current format at the full
// view's width.
func (m *dashModel) detailNodeLines() []string {
e := m.entries[m.cursor]
lines, _ := dashNodeView(e.name, m.results[m.cursor], m.actions[m.cursor],
dashNow(), dashStaleAfter(e.kind))
dashNow(), dashStaleAfter(e.kind), m.gauge, barLineW)
return lines
}

Expand Down
15 changes: 13 additions & 2 deletions cmd/spinloop/dashboard_model.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@ type dashModel struct {
confirm bool // a stop is waiting on its confirmation
statusLine string

// gauge is the board's resource-series format: false draws the bar
// format (the sparkline of each node's retained history), true the gauge
// format (the current reading). Board-wide, toggled by g — one format for
// every panel rather than a choice per node. The zero value opens the
// board in bar.
gauge bool

// send feeds a message back into the program from outside the Update
// loop — the in-flight progress of a start, which its call reports from
// its own goroutine. It is the tea.Program's Send, safe from any
Expand Down Expand Up @@ -351,6 +358,10 @@ func (m *dashModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
if len(m.entries) > 0 && m.actions[m.cursor].verb == "" {
m.confirm = true
}
case "g":
// The board-wide format toggle: every panel redraws in the other
// format, and g again returns them.
m.gauge = !m.gauge
case "r":
// A manual refresh is due for every node, cloud or local,
// whatever their own deadlines say.
Expand Down Expand Up @@ -658,7 +669,7 @@ func (m dashModel) View() string {
tiles := make([]string, len(m.entries))
for i := range m.entries {
tiles[i] = dashTile(m.entries[i].name, m.results[i], i == m.cursor, m.actions[i],
now, dashStaleAfter(m.entries[i].kind))
now, dashStaleAfter(m.entries[i].kind), m.gauge)
}
rows := dashGridRows(tiles, dashCols(w))
lo := m.scrollRow
Expand Down Expand Up @@ -688,7 +699,7 @@ func (m dashModel) headerLine(w int) string {

// dashGridKeys is the grid's own key help; the detail view's footer shares
// footerLine but names its own keys instead (see dashDetailKeys).
const dashGridKeys = "↑↓←→ move s start a abort x stop r refresh q quit"
const dashGridKeys = "↑↓←→ move s start a abort x stop g format r refresh q quit"

// footerLine is the frame's bottom line: the given key help, replaced by the
// stop confirmation prompt while one is pending, with the status line and a
Expand Down
29 changes: 20 additions & 9 deletions cmd/spinloop/dashboard_render.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ func dashStaleAfter(kind string) time.Duration {
// its age wherever it is drawn, and takes the panel to the unknown tier: a
// stale reading is not a wrong reading, but drawing it identically to a
// current one is.
func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time, staleAfter time.Duration) ([]string, dashHealthTier) {
func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time, staleAfter time.Duration, gauge bool, lineW int) ([]string, dashHealthTier) {
age := dashReadingAge(r, now, staleAfter)
var b strings.Builder
switch {
Expand All @@ -297,7 +297,7 @@ func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time,
if s := r.Metrics.State; s != "" {
fmt.Fprintln(&b, dashStateLine(s, r.Metrics)+age)
}
dashTileReportBody(&b, r.Metrics, true)
dashTileReportBody(&b, r.Metrics, true, gauge, lineW)
}
case r.Outcome == "":
fmt.Fprintf(&b, "%s\nwaiting for first refresh…\n", name)
Expand All @@ -308,7 +308,14 @@ func dashNodeView(name string, r fleet.NodeResult, a dashAction, now time.Time,
}
default:
fmt.Fprintf(&b, "%s %s%s\n", name, dashStateLine(r.Metrics.State, r.Metrics), age)
dashTileReportBody(&b, r.Metrics, r.Metrics.State == "running")
// In bar the series also come from the retained history, which
// survives a stop: a stopped node with a window still has series to
// draw, and its current reading simply carries no fallback for them.
resources := r.Metrics.State == "running"
if !gauge && len(r.Metrics.History) > 0 {
resources = true
}
dashTileReportBody(&b, r.Metrics, resources, gauge, lineW)
}
lines := strings.Split(b.String(), "\n")
lines = lines[:len(lines)-1] // the trailing newline splits an extra empty piece
Expand Down Expand Up @@ -373,8 +380,8 @@ func dashHealthTierFor(r fleet.NodeResult, a dashAction, stale bool) dashHealthT
// tile's fixed height and clipped to its fixed width, with the first line
// drawn as the header bar — tile-only, not part of dashNodeView, so the detail
// view (which draws the same lines full-screen) keeps a plain first line.
func dashTileContent(name string, r fleet.NodeResult, a dashAction, now time.Time, staleAfter time.Duration) string {
lines, tier := dashNodeView(name, r, a, now, staleAfter)
func dashTileContent(name string, r fleet.NodeResult, a dashAction, now time.Time, staleAfter time.Duration, gauge bool) string {
lines, tier := dashNodeView(name, r, a, now, staleAfter, gauge, dashBarLineW)
if len(lines) == 0 {
lines = []string{""}
}
Expand Down Expand Up @@ -426,19 +433,23 @@ func dashStateLine(state string, m metrics.Stats) string {
// answer has it. A settled tile gates the resources block on the node being
// running; the in-flight tile draws whatever there is, because a boot half
// done has some of the facts and not the rest.
func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool) {
func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool, lineW int) {
if line := dashTileServingLine(m); line != "" {
fmt.Fprintln(w, line)
}
renderLastActiveIndented(w, m.LastActiveAt, m.IdleSeconds)
if resources {
renderStatBars(w, m.CPU, m.Memory, m.GPUs)
if gauge {
renderStatGauges(w, m.CPU, m.Memory, m.GPUs)
} else {
renderStatBars(w, m.CPU, m.Memory, m.GPUs, m.History, lineW)
}
renderTokenLines(w, m.Tokens)
}
}

// dashTile frames one panel; the selected one carries a lit border.
func dashTile(name string, r fleet.NodeResult, selected bool, a dashAction, now time.Time, staleAfter time.Duration) string {
func dashTile(name string, r fleet.NodeResult, selected bool, a dashAction, now time.Time, staleAfter time.Duration, gauge bool) string {
style := lipgloss.NewStyle().
Width(dashTileW).Height(dashTileH).
Border(lipgloss.RoundedBorder())
Expand All @@ -447,7 +458,7 @@ func dashTile(name string, r fleet.NodeResult, selected bool, a dashAction, now
} else {
style = style.BorderForeground(lipgloss.Color("240"))
}
return style.Render(dashTileContent(name, r, a, now, staleAfter))
return style.Render(dashTileContent(name, r, a, now, staleAfter, gauge))
}

// dashGridRows lays tiles out left to right, top to bottom, in fleet-file
Expand Down
29 changes: 20 additions & 9 deletions cmd/spinloop/fleet.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ func fleetMetricsCmd() *cobra.Command {
SilenceUsage: true,
RunE: func(c *cobra.Command, _ []string) error {
resolve(c)
if format != "bar" && format != "table" && format != "json" {
return fmt.Errorf("--format must be \"bar\", \"table\", or \"json\", got %q", format)
if err := validateMetricsFormat(format); err != nil {
return err
}
cfg, err := fleet.Resolve(path)
if err != nil {
Expand All @@ -134,7 +134,7 @@ func fleetMetricsCmd() *cobra.Command {
}
fs := c.Flags()
fs.StringVarP(&path, "fleet", "f", "", fleetFileUsage)
fs.StringVar(&format, "format", "bar", "output format: bar (default), table or json")
fs.StringVar(&format, "format", "bar", "output format: bar (default), gauge, table or json")
fs.BoolVarP(&watch, "watch", "w", false, "redraw the fleet every 60 seconds")
c.ValidArgsFunction = noPositionals
compRegister(c, "fleet", compFiles)
Expand Down Expand Up @@ -205,13 +205,24 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string)
// before theirs: a node whose engine has stopped still has a useful
// answer to "when did this last do anything?".
renderLastActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds)
if stats.State != "running" {
continue
}
if format == "bar" {
renderStatBars(w, stats.CPU, stats.Memory, stats.GPUs)
switch format {
case "bar":
// No state gate, for the same reason the remote bar format has
// none: a stopped node's retained history says what its engine
// was doing until it stopped, and a stopped node's current
// reading carries no figures for it to fall back on.
renderStatBars(w, stats.CPU, stats.Memory, stats.GPUs, stats.History, barLineW)
renderTokenLines(w, stats.Tokens)
} else {
case "gauge":
if stats.State != "running" {
continue
}
renderStatGauges(w, stats.CPU, stats.Memory, stats.GPUs)
renderTokenLines(w, stats.Tokens)
default:
if stats.State != "running" {
continue
}
renderTokenLines(w, stats.Tokens)
renderGPUTable(w, stats.GPUs)
renderCPUMemTable(w, stats.CPU, stats.Memory)
Expand Down
128 changes: 120 additions & 8 deletions cmd/spinloop/fleet_dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -288,11 +288,13 @@ func dashFixNow(t *testing.T, at time.Time) {
t.Cleanup(func() { dashNow = time.Now })
}

// dashTestTile draws one tile at the board's current clock, with the staleness
// bound of a local node — the tile tests supply readings with no time on them,
// which are never called stale, so the bound is not what any of them is about.
// dashTestTile draws one tile at the board's current clock, in the board's
// default format (bar), with the staleness bound of a local node — the tile
// tests supply readings with no time on them, which are never called stale, so
// the bound is not what any of them is about. A test that wants the gauge
// format draws with dashTile directly.
func dashTestTile(name string, r fleet.NodeResult, selected bool, a dashAction) string {
return dashTile(name, r, selected, a, dashNow(), dashStaleAfter(fleet.KindDaemon))
return dashTile(name, r, selected, a, dashNow(), dashStaleAfter(fleet.KindDaemon), false)
}

// dashExpectedHeader is the tile's header bar as a test spells it out: the
Expand Down Expand Up @@ -607,7 +609,7 @@ func TestDashNodeViewEveryPhaseAgainstEveryReading(t *testing.T) {
for _, rd := range readings {
t.Run(ph.name+" over "+rd.name, func(t *testing.T) {
a := dashAction{verb: "start", since: now.Add(-30 * time.Second), phase: ph.phase}
lines, tier := dashNodeView("n", rd.r, a, now, staleAfter)
lines, tier := dashNodeView("n", rd.r, a, now, staleAfter, false, barLineW)
joined := strings.Join(lines, "\n")
// The action's own account leads, and the node's report
// follows it where the reading has one to give.
Expand All @@ -633,7 +635,7 @@ func TestDashNodeViewEveryPhaseAgainstEveryReading(t *testing.T) {
}
// Nothing about the action is shown once it settles, and the
// reading alone then decides the tier.
settledLines, settledTier := dashNodeView("n", rd.r, dashAction{}, now, staleAfter)
settledLines, settledTier := dashNodeView("n", rd.r, dashAction{}, now, staleAfter, false, barLineW)
if settledTier != rd.settled {
t.Errorf("settled tier = %v, want %v", settledTier, rd.settled)
}
Expand All @@ -658,7 +660,7 @@ func TestDashTileStaleReadingShowsItsAgeAndRecovers(t *testing.T) {
Metrics: metrics.Stats{State: "running", Ready: "ready"},
At: now.Add(-4 * time.Minute)}
staleAfter := dashStaleAfter(fleet.KindRemote) // three minutes, on the minute cadence
got := dashTile("dev-1", r, false, dashAction{}, now, staleAfter)
got := dashTile("dev-1", r, false, dashAction{}, now, staleAfter, false)
want := dashTileExpected([]string{
dashExpectedHeader("dev-1 running · 4m 0s ago", dashUnknown),
"", "", "", "", "", "", "", "", "", "", "",
Expand All @@ -672,7 +674,7 @@ func TestDashTileStaleReadingShowsItsAgeAndRecovers(t *testing.T) {
dashExpectedHeader("dev-1 running", dashHealthy),
"", "", "", "", "", "", "", "", "", "", "",
})
if got := dashTile("dev-1", r, false, dashAction{}, now, staleAfter); got != wantFresh {
if got := dashTile("dev-1", r, false, dashAction{}, now, staleAfter, false); got != wantFresh {
t.Errorf("recovered tile mismatch:\ngot:\n%q\nwant:\n%q", got, wantFresh)
}
}
Expand Down Expand Up @@ -3076,3 +3078,113 @@ func TestDashProgramDetailViewLogAndBack(t *testing.T) {
t.Fatalf("starts=%d, want 1", node.starts)
}
}

// A node with retained readings, for the format tests: the same engine a
// byte-stability test would draw, with a short history behind its current
// reading.
func dashHistoryNode() fleet.NodeResult {
return fleet.NodeResult{
Name: "up", Outcome: fleet.OutcomeOK,
Metrics: metrics.Stats{
State: "running", Runner: "llamacpp", ModelID: "org/qwen:q4",
UptimeSeconds: 7200, LastActiveAt: "2026-08-21T10:00:00Z", IdleSeconds: 12,
CPU: &metrics.CpuStat{Utilization: 42},
Memory: &metrics.MemoryStat{Total: 1000, Used: 300},
GPUs: []metrics.GpuStat{{Index: 0, Name: "H100", Utilization: 61, MemoryUsed: 80, MemoryTotal: 160}},
Tokens: &metrics.TokenStats{Running: 2, PromptTokens: 4096, GenerationTokens: 1024, Requests: 17},
History: []metrics.HistorySample{
{Time: 1786276800, CPU: ptrPct(10), Mem: ptrPct(20), GPUs: []metrics.HistoryGPU{{Index: 0, Util: 50, Mem: ptrPct(40)}}},
{Time: 1786276815, CPU: ptrPct(20), Mem: ptrPct(30), GPUs: []metrics.HistoryGPU{{Index: 0, Util: 61, Mem: ptrPct(50)}}},
},
},
}
}

func dashTileAt(gauge bool) string {
return dashTile("up", dashHistoryNode(), false, dashAction{}, dashNow(), dashStaleAfter(fleet.KindDaemon), gauge)
}

// The g key toggles the board-wide format: every panel redraws in the other
// format, and g again returns them. The formats differ only where a series
// has history to draw — a node without it looks the same either way.
func TestDashModelFormatToggle(t *testing.T) {
lipgloss.SetColorProfile(termenv.Ascii)
dashFixNow(t, dashTestClock)
r := dashHistoryNode()
m := &dashModel{
entries: []dashEntry{{name: "up"}},
results: []fleet.NodeResult{r},
actions: []dashAction{{}},
width: 120, height: 40,
}

bar := dashTileAt(false)
gauge := dashTileAt(true)
if bar == gauge {
t.Fatal("the two formats drew identical tiles for a node with history")
}
// The bar format draws the retained readings: a fresh window, so leading
// blank columns, ending on the last retained sample — the current reading
// is the next point, not in the history yet.
cpuLine := " CPU " + strings.Repeat(" ", 23) + "▁" + ansiGreen + "▂" + ansiReset + " 20%"
if !strings.Contains(bar, cpuLine) {
t.Errorf("bar tile did not draw the history, want %q in:\n%s", cpuLine, bar)
}
// The gauge format draws the current reading filled, as before the change.
if !strings.Contains(gauge, dashBar("CPU", 42)) || strings.Contains(gauge, "▁") {
t.Errorf("gauge tile: %q", gauge)
}

// One press flips the flag, the second press returns it.
next, cmd := m.Update(dashKey("g"))
if cmd != nil {
t.Fatal("the format toggle returned a cmd")
}
m = next.(*dashModel)
if !m.gauge {
t.Fatal("g did not switch the board to gauge")
}
next, _ = m.Update(dashKey("g"))
m = next.(*dashModel)
if m.gauge {
t.Fatal("a second g did not return the board to bar")
}
}

// Both formats keep the tile's geometry: the frame is the same width and
// height whatever the body draws, so the board's grid never shifts.
func TestDashTileGeometryHoldsInBothFormats(t *testing.T) {
lipgloss.SetColorProfile(termenv.Ascii)
dashFixNow(t, dashTestClock)
for _, gauge := range []bool{false, true} {
lines := strings.Split(dashTileAt(gauge), "\n")
if len(lines) != dashTileH+2 {
t.Errorf("gauge=%v: %d lines, want %d", gauge, len(lines), dashTileH+2)
continue
}
for i, line := range lines {
if w := lipgloss.Width(line); w != dashTileW+2 {
t.Errorf("gauge=%v line %d: width %d, want %d", gauge, i, w, dashTileW+2)
}
}
}
}

// A node the daemon has not filled the history for — old daemons, a series
// the engine never reported — falls back to the gauge drawing in the bar
// format, so it renders exactly as it did before the change.
func TestDashTileBarFallsBackToGaugeWithoutHistory(t *testing.T) {
lipgloss.SetColorProfile(termenv.Ascii)
dashFixNow(t, dashTestClock)
r := dashHistoryNode()
r.Metrics.History = nil
tile := func(gauge bool) string {
return dashTile("up", r, false, dashAction{}, dashNow(), dashStaleAfter(fleet.KindDaemon), gauge)
}
if got, want := tile(false), tile(true); got != want {
t.Errorf("a history-less node differs between formats:\nbar:\n%q\ngauge:\n%q", got, want)
}
if !strings.Contains(tile(false), dashBar("CPU", 42)) {
t.Errorf("the fallback did not draw the old gauges: %q", tile(false))
}
}
Loading